diff --git a/.gitignore b/.gitignore index a77e2a5a5..0c5143875 100644 --- a/.gitignore +++ b/.gitignore @@ -213,3 +213,6 @@ CLAUDE.local.md # Generated dataset cache (created by Dataset.get_dataloader()) dataset_cache/ .agents/devdocs/ + +# Run artifacts: the example configs' default report_dir +logs/ diff --git a/AGENTS.md b/AGENTS.md index e266776dd..e9a11d01e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -101,6 +101,7 @@ Dataset Manager --> Load Generator --> Endpoint Client --> External Endpoint | **DeepSeek-R1 (MLPerf)** | `src/inference_endpoint/evaluation/scoring.py` (`LegacyMLPerfDeepSeekR1Scorer`), `examples/07_DeepSeekR1_Example/` | MLPerf DeepSeek-R1 accuracy. TensorRT-LLM is OpenAI-compatible, so it is served via `api_type: openai` / `openai_completions` (no dedicated trtllm adapter). The combined multi-subset eval (`math500`/`aime`/`gpqa`/`mmlu_pro`/`livecodebench`) is the official MLCommons `eval_accuracy.py`, run out-of-process via `uv run --project` against the isolated subproject at `src/inference_endpoint/evaluation/legacy_mlperf_deepseek_r1/` (a uv subproject excluded from the parent wheel; mirrors the VBench pattern). The example feeds the exact MLPerf prompt via pre-tokenized `input_tokens` to `/v1/completions`. | | **VideoGen** | `src/inference_endpoint/videogen/` | Adapter for video-generation endpoints (e.g. trtllm-serve `POST /v1/videos/generations`, used by MLPerf WAN2.2-T2V-A14B). Defaults to `response_format=video_path` (server saves video to shared storage and returns path) to avoid large byte payloads. Accuracy mode also runs on `video_path`: the adapter mirrors the path into `response_output` so the event log carries it to `VBenchScorer` (see `evaluation/scoring.py`), which scores videos via VBench from a sibling `uv` subproject at `examples/09_Wan22_VideoGen_Example/accuracy/` (vbench's `transformers==4.33.2` + `numpy<2` pins are incompatible with the parent env, so it runs out-of-process via `uv run --project`). Dataset is ingested via the generic JSONL loader. | | **SWE-bench** | `src/inference_endpoint/dataset_manager/predefined/swe_bench/`, `src/inference_endpoint/evaluation/swe_bench_scorer.py`, `src/inference_endpoint/evaluation/swebench_service/` | `SWEBench` predefined dataset (HuggingFace `princeton-nlp/SWE-bench_Verified` or `_Lite`; `ACCURACY_ONLY=True`). `SWEBenchScorer` sets `SKIP_ENDPOINT_PHASE=True` and bypasses the built-in accuracy phase entirely: it delegates agent execution and grading to the configured SWE-bench service via `accuracy_config.extras.swebench_service_url`. The service is an isolated `uv` subproject; its host owns Docker/runtime execution, artifacts, and credentials, while the benchmark client remains the report-producing entrypoint. | +| **Whisper (MLPerf speech2text)** | `src/inference_endpoint/openai/transcriptions_adapter.py`, `dataset_manager/predefined/whisper_librispeech/`, `evaluation/wer.py` | MLPerf speech2text (whisper-large-v3 / LibriSpeech). `api_type: openai_transcriptions` posts a `multipart/form-data` upload to `/v1/audio/transcriptions`, the only route serving Whisper on vLLM, so the adapter declares `HttpRequestAdapter.CONTENT_TYPE` and the worker folds it into the request prefix at setup. `Query.data` carries the clip's _path_; the worker reads the bytes in `encode_query`, so the corpus never crosses the ZMQ frame. Non-streaming. `whisper_librispeech` fetches the MLCommons-prepared `dev-all-repack.json` via its own `_download_from_r2`; the repackaging stays upstream because clip boundaries change the WER. `WERScorer` (`eval_method: wer`) reports `100 - WER%` from the ported metric in `evaluation/wer.py`. Reference numbers live in `models.Whisper_large_v3` / `datasets.LibriSpeechDevAll`. | | **Compliance (submission checker)** | `src/inference_endpoint/compliance/checker.py`, `scripts/check_compliance.py` | Validates a completed run's report directory against a registered ruleset. `check_submission(report_dir, ruleset, model)` reads the resolved `config.yaml` plus scorer output (`accuracy/accuracy_results.json` for accuracy, `scores.json` for the agentic perf run) and runs config-lock (deterministic + single-stream), the accuracy gate (`score >= factor x reference`, factor 0.97 for Edge-Agentic), and run validity (0 dropped turns). Server-side launch flags (`--reasoning off`, `--ctx-size`) aren't in client artifacts, so they're surfaced as manual attestations. CLI: `scripts/check_compliance.py REPORT_DIR` (exit 0 = pass). | | **Compliance (audit tests)** | `src/inference_endpoint/compliance/`, `commands/audit.py` | MLPerf compliance audits. `AuditTest` protocol + `AuditRunSpec`/`AuditRunArtifacts` + registry (`compliance/__init__.py`); `OutputCachingAudit` (`compliance/audit_test/output_caching_test.py`, which also owns the QPS-specific `AuditRunStats`) implements MLPerf **TEST04** output-caching detection — reference phase (distinct samples) vs. fixed-sample audit phase, comparing QPS against `threshold`. `commands/audit.py:run_audit` runs phases via `AuditTest.plan_runs`/`validate`, writing `audit_result.json`/`verify_.txt` atomically via `compliance/result.py`. Enabled by the `audit:` YAML block; `cli._run` runs it after the main benchmark (upstream MLPerf order: perf run, then TEST04), or standalone with `audit.only: true`. Perf-only by default (a phase may opt into accuracy via `AuditRunSpec.test_mode`, but this is unused today). | | **Validation** | `src/inference_endpoint/validation/`, `docs/validation/` | Cohort checker for submission uploads. A cohort is a directory of YAML checks (four scope files + `valid_refs.yaml` + a `cohort.yaml` naming the packaged ruleset); `python -m inference_endpoint.validation COHORT SUBMISSION` runs them against an upload root (`results///r/`) and reports holds/failures per check. Exit codes: 0 holds, 1 blocking failure, 2 refusal. See docs/validation/README.md. | @@ -357,7 +358,7 @@ src/inference_endpoint/ │ ├── output_caching_test.py # OutputCachingAudit (MLPerf TEST04): caching detection (reference vs fixed-sample QPS) │ └── README.md # TEST04 output-caching audit usage (WAN 2.2 example) ├── core/ -│ ├── types.py # APIType, Query, QueryResult, StreamChunk, QueryStatus (msgspec Structs) +│ ├── types.py # APIType (default_route + supports_streaming/has_text_prompt capabilities), Query, QueryResult, StreamChunk, QueryStatus (msgspec Structs) │ └── record.py # EventRecord — transport record used by event logger and ZMQ transport ├── load_generator/ │ ├── session.py # BenchmarkSession, PhaseIssuer, PhaseConfig, PhaseResult, SessionResult @@ -399,7 +400,7 @@ src/inference_endpoint/ │ ├── dataset.py # Dataset base class, DatasetFormat enum │ ├── factory.py # Dataset factory │ ├── transforms.py # ColumnRemap and other transforms -│ └── predefined/ # Built-in datasets (aime25, cnndailymail, gpqa, etc.) +│ └── predefined/ # Built-in datasets (aime25, cnndailymail, gpqa, whisper_librispeech, etc.) ├── metrics/ │ ├── report.py # Report.from_snapshot(MetricsSnapshot); display + JSON serialization │ ├── metric.py # Metric types (Throughput, etc.) @@ -415,10 +416,11 @@ src/inference_endpoint/ │ ├── rulesets/mlcommons/ # MLCommons-specific rules, datasets, models │ └── templates/ # YAML config templates (_template.yaml minimal, _template_full.yaml all defaults) ├── openai/ # OpenAI-compatible API types and adapters -│ ├── types.py # OpenAI response types (chat + text completion) +│ ├── types.py # OpenAI response types (chat + text completion + transcription) │ ├── openai_adapter.py # Chat completions adapter (/v1/chat/completions) │ ├── openai_msgspec_adapter.py # msgspec-based chat completions adapter (fast path) │ ├── completions_adapter.py # Text completions adapter (/v1/completions, pre-tokenized input) +│ ├── transcriptions_adapter.py # Audio transcription adapter (/v1/audio/transcriptions, multipart upload) │ ├── accumulator.py # Streaming response accumulator (shared by chat + completions) │ └── harmony.py # openai_harmony integration ├── sglang/ # SGLang API adapter (/generate with input_ids) @@ -427,6 +429,7 @@ src/inference_endpoint/ │ ├── types.py # Pydantic: VideoPathRequest, VideoPathResponse, VideoPayloadResponse │ └── adapter.py # VideoGenAdapter (HttpRequestAdapter) + VideoGenAccumulator (no-op) ├── evaluation/ # Accuracy evaluation (extractor, scoring, livecodebench) +│ ├── wer.py # MLPerf speech2text Word Error Rate (ported metric; used by WERScorer) │ └── swebench_service/ # Isolated uv service for Docker/Pyxis SWE-bench runs ├── compliance/ # Submission compliance checks (config-lock, accuracy gate, run validity) │ ├── __init__.py diff --git a/docs/ENDPOINT_CLIENT.md b/docs/ENDPOINT_CLIENT.md index 72c015d5d..43cc85a74 100644 --- a/docs/ENDPOINT_CLIENT.md +++ b/docs/ENDPOINT_CLIENT.md @@ -280,6 +280,7 @@ model validator. ```python class APIType(str, Enum): OPENAI = "openai" + OPENAI_TRANSCRIPTIONS = "openai_transcriptions" SGLANG = "sglang" class HTTPClientConfig(BaseModel): @@ -960,19 +961,19 @@ concatenating them with a single `b"".join()`. | Method | Async | Description | | ------------------------------------------------ | ----- | -------------------------------------------------------------------------- | -| `from_url(host, port, path)` | No | Classmethod: create template with pre-encoded request line + Host header | +| `from_url(host, port, path, content_type?)` | No | Classmethod: create template with pre-encoded request line + Host header | | `cache_headers(headers)` | No | Pre-encode headers (e.g. Authorization) into bytes; call once during setup | | `build_request(body, streaming, extra_headers?)` | No | Build complete HTTP/1.1 request bytes; fast path when no extra headers | **Request byte segments:** -| Segment | When Built | Description | -| -------------- | ------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Static prefix | Once (init) | `POST /v1/chat/completions HTTP/1.1\r\nHost: localhost:8000\r\n` — pre-encoded at construction via `from_url()` | -| Cached headers | Once (setup) | `Authorization: Bearer sk-...\r\n` — pre-encoded via `cache_headers()`; reused across all requests | -| Content-Type | Once (class) | Two class-level `bytes` constants: `HEADERS_STREAMING` and `HEADERS_NON_STREAMING`. The streaming/non-streaming branch selects between them — no string encoding at request time | -| Content-Length | Per-request | `Content-Length: 1234\r\n\r\n` — the only header that changes per request; built from a single f-string and ASCII-encoded | -| Body | Per-request | JSON bytes produced by the adapter's `encode_query()` ([§9.1](#91-httprequestadapter)) | +| Segment | When Built | Description | +| -------------- | ------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Static prefix | Once (init) | `POST /v1/chat/completions HTTP/1.1\r\nHost: localhost:8000\r\n` — pre-encoded at construction via `from_url()` | +| Cached headers | Once (setup) | `Authorization: Bearer sk-...\r\n` — pre-encoded via `cache_headers()`; reused across all requests | +| Content-Type | Once (setup) | From the adapter's `CONTENT_TYPE` (default `application/json`; `openai_transcriptions` declares a `multipart/form-data` boundary). Folded into two prefixes with the matching `Accept`, one per streaming branch — no string encoding at request time | +| Content-Length | Per-request | `Content-Length: 1234\r\n\r\n` — the only header that changes per request; built from a single f-string and ASCII-encoded | +| Body | Per-request | Bytes produced by the adapter's `encode_query()` ([§9.1](#91-httprequestadapter)), encoded in the adapter's `CONTENT_TYPE` | ![build_request() fast/slow path](res/endpoint_client/11_build_request_flow.png) diff --git a/docs/evaluation/DESIGN.md b/docs/evaluation/DESIGN.md index 7e62c4ee7..6f9113321 100644 --- a/docs/evaluation/DESIGN.md +++ b/docs/evaluation/DESIGN.md @@ -82,6 +82,7 @@ The scorer registry in `evaluation/scoring.py` currently includes: | `vbench` | VBench video-generation accuracy (WAN 2.2 T2V) | | `bfcl_v4` | BFCL v4 function-calling accuracy | | `legacy_mlperf_deepseek_r1` | MLPerf DeepSeek-R1 combined multi-subset accuracy | +| `wer` | MLPerf speech2text Word Error Rate (Whisper / LibriSpeech) | The registry (`Scorer.PREDEFINED`) is auto-populated from `Scorer` subclasses via `__init_subclass__`. diff --git a/examples/12_Whisper_Example/README.md b/examples/12_Whisper_Example/README.md new file mode 100644 index 000000000..a111babbc --- /dev/null +++ b/examples/12_Whisper_Example/README.md @@ -0,0 +1,171 @@ +# MLPerf speech2text — whisper-large-v3 on LibriSpeech + +Benchmarks an OpenAI-compatible transcription endpoint +(`POST /v1/audio/transcriptions`) against the MLPerf +[speech2text](https://github.com/mlcommons/inference/tree/master/speech2text) +workload and scores Word Error Rate. + +| | | +| ---------------- | --------------------------------------------------------------------------- | +| Model | `openai/whisper-large-v3` | +| Dataset | LibriSpeech dev-clean + dev-other, repackaged to ~30 s clips (1633 samples) | +| Metric | WER (reported as `100 - WER%`) | +| Reference | WER 2.0671% → accuracy **97.9329%** | +| Submission floor | 99% of reference → **96.9536** | +| Scenario | Offline (`max_throughput`) | + +## 1. Serve the model + +```bash +vllm serve openai/whisper-large-v3 --port 8000 +``` + +Whisper is an encoder-decoder ASR model, so it is served only on +`/v1/audio/transcriptions` — not `/v1/chat/completions`. + +## 2. The dataset + +Nothing to do: the first run downloads the MLCommons-prepared corpus into +`dataset_cache/whisper_librispeech/source/` (several GB of audio) and caches the +parsed rows as a parquet beside it. The ~30 s repackaging is part of the +benchmark definition, so the prepared artifact is fetched rather than rebuilt +here: different clip boundaries would produce a WER that is not comparable to +the reference. + +``` +/dev-all-repack.json # manifest: files[0].fname, original_duration, transcript +/dev-all-repack/*.wav # 16 kHz mono clips +``` + +The manifest's `fname` entries keep the upstream pipeline's `./data/` prefix, +which the hosted artifact does not reproduce. The loader probes the first clip +and drops any leading components that aren't present under the root, so both +layouts resolve with no configuration. + +To use a copy you already have, set `datasets[*].generate_params.manifest` or +`$WHISPER_LIBRISPEECH_MANIFEST`; either skips the download. +`generate_params.data_dir` overrides the root that `fname` resolves against; it +defaults to the manifest's own directory. + +The parsed-row cache is keyed on the dataset ID alone, so if you change +`manifest` or `data_dir` after a first run, also pass +`generate_params.force: true` once — otherwise the rows built from the previous +manifest are reused. + +Keep the WAVs on **fast local storage**. The worker reads each file inside the +measured request, and that read is synchronous on its event loop: a slow read +stalls every other in-flight request in the same worker process, so a network +mount or cold cache inflates latency beyond the clip that caused it. + +## 3. Run + +Edit `endpoint_config.endpoints` first. The config issues the full 1633-sample +corpus in both the performance and accuracy phases, which is what a submission +needs. + +```bash +uv run inference-endpoint benchmark from-config \ + --config examples/12_Whisper_Example/offline_whisper.yaml +``` + +To check the wiring first, set `runtime.n_samples_to_issue` and both `samples` +values to 8. The dataset truncates in manifest order, so the smoke subset is a +prefix of the full corpus — but a partial perf phase is not a reportable result. + +## 4. Read the result + +`/accuracy/accuracy_results.json`: + +```json +{ + "score": 98.1, + "complete": true, + "breakdown": { + "wer_percent": 1.897, + "edit_distance": 1993, + "reference_words": 105077, + "empty_responses": 0, + "missing_samples": 0 + } +} +``` + +A submission needs `complete: true` and `missing_samples: 0`. An issued sample +that never produced a response is scored as an empty transcript — its reference +words count in the denominator and all of them count as errors — so a partial +run shows up as a depressed score rather than as a flattering one, and +`complete` goes `false`. `empty_responses` is the separate case of a sample that +did answer, with nothing in it. + +Those are the figures from a healthy full-corpus run; `reference_words` is fixed +at 105077 by the corpus, so a different value there means the manifest is not +the repackaged `dev-all-repack.json`. A WER well above the reference 2.0671% +points at the dataset rather than the endpoint for the same reason. + +## How it maps onto the framework + +| Piece | Where | +| ------------------------ | ---------------------------------------------------------------------- | +| Route + multipart upload | `openai/transcriptions_adapter.py` (`api_type: openai_transcriptions`) | +| Dataset | `dataset_manager/predefined/whisper_librispeech/` | +| Metric | `evaluation/wer.py` + `WERScorer` (`eval_method: wer`) | +| Reference numbers | `config/rulesets/mlcommons/models.py` (`Whisper_large_v3`) | + +### Notes on the run config + +The first three are enforced by config validation keyed on +`api_type: openai_transcriptions`, so a hand-written config gets an error rather +than a silently odd run. The fourth is an adapter default, not a constraint: + +- **Non-streaming.** OpenAI does not support `stream` for Whisper-class models. + `streaming: on` is rejected and `auto` resolves to `off` even in online mode. + There is no TTFT/TPOT; the metrics are QPS and per-request latency. +- **`metrics_isl` is forced off.** Audio rows carry no text prompt, so there is + no input sequence length. One "ISL is unavailable" warning per run is + expected. +- **Warmup salt is rejected.** Salting prepends a random hex prefix to a text + prompt to defeat prefix caching; an audio upload has no prompt, and its cache + key is the audio. +- **`language: en`** is sent on every request (the adapter's `DEFAULT_LANGUAGE`; + a dataset `language` column overrides it). It makes the server build Whisper's + `<|startoftranscript|><|en|><|transcribe|><|notimestamps|>` prompt instead of + auto-detecting the language per clip, which is what the MLPerf reference does. + +## Reference numbers + +The reference accuracy, the 0.99 factor and the 1633-sample count are registered +as reference data in `config/rulesets/mlcommons/models.py` (`Whisper_large_v3`) +and `datasets.py` (`LibriSpeechDevAll`), so a `submission_ref` run resolves them +from the ruleset instead of the numbers living only in this file. They +transcribe the MLPerf Inference submission checker's own constants, where the +gate is `"whisper": ("ACCURACY", (100.0 - 2.0671) * 0.99)` — the same +`100 - WER` scale `WERScorer` reports on. + +The in-repo `compliance/checker.py` does **not** cover this workload: its +accuracy gate resolves BFCL metric keys only, and its config-lock expects a +single-stream run (`num_workers: 1`), which this example is not. Checking a +whisper submission against the gate is the external MLPerf Inference submission +checker's job. + +### Expect a score a little above the reference + +A healthy run scores slightly better than the reference, not equal to it. The +scoring is not the reason: `evaluation/wer.py` reproduces +`compute_wer_with_concatenation` from the MLCommons reference with zero +per-sample disagreement over the full corpus, and the two candidate +`EnglishTextNormalizer` implementations (`transformers` and `openai-whisper`) +give the same corpus WER. The difference is the serving path, and it is inherent +to benchmarking an endpoint: + +| | MLPerf reference SUT | This example | +| -------------- | ---------------------------------------------------------------------------- | ------------------------------------------------------ | +| Engine | vLLM V0, offline `LLM.generate()` | any OpenAI-compatible server, over HTTP | +| Decoder prompt | explicit `<\|startoftranscript\|><\|en\|><\|transcribe\|><\|notimestamps\|>` | built by the server's `/v1/audio/transcriptions` layer | +| Audio decode | `librosa.load(path, sr=16000)` in-process | server-side, from the multipart upload | +| Sampling | `temperature=0, top_p=1.0, max_tokens=200` | `temperature=0`, length left to the server | + +That reference's own SUT pins `dtype = "bfloat16"`, which is why +`Whisper_large_v3.golden_accuracy` is registered as `bfloat16` rather than fp32. +Serving fp32 instead moves the score by well under 0.1 points, so a gap of that +size is engine variance rather than a scoring fault. `max_tokens=200` never +binds in practice — observed output lengths top out near 150 tokens. diff --git a/examples/12_Whisper_Example/offline_whisper.yaml b/examples/12_Whisper_Example/offline_whisper.yaml new file mode 100644 index 000000000..72f2a8b29 --- /dev/null +++ b/examples/12_Whisper_Example/offline_whisper.yaml @@ -0,0 +1,59 @@ +# Offline run for MLPerf speech2text (whisper-large-v3 / LibriSpeech). +# +# Needs a server on /v1/audio/transcriptions: +# vllm serve openai/whisper-large-v3 --port 8000 +# +# The corpus downloads itself on first run. See README.md for the reference +# accuracy, the storage requirement for the WAVs, how to point at a local +# manifest, and what api_type validation derives rather than reading here. +# +# For a smoke run, set runtime.n_samples_to_issue and both `samples` to 8. + +name: "offline-whisper-librispeech" +version: "1.0" +type: "offline" + +# Streaming, metrics_isl and warmup salt are all settled by api_type +# validation, so they are deliberately absent below rather than set to the +# value they would be forced to. README.md ("Notes on the run config") lists +# what is derived and why. +model_params: + name: "openai/whisper-large-v3" + temperature: 0.0 + +datasets: + # setup_benchmark() requires a performance dataset to drive issuance; the + # accuracy entry below reuses the same corpus and scores the transcripts. + - name: whisper_librispeech + type: "performance" + samples: 1633 + - name: whisper_librispeech + type: "accuracy" + samples: 1633 + accuracy_config: + eval_method: "wer" + ground_truth: "transcript" + num_repeats: 1 + +settings: + runtime: + n_samples_to_issue: 1633 + scheduler_random_seed: 42 + dataloader_random_seed: 42 + + load_pattern: + type: "max_throughput" + + client: + num_workers: 8 + + warmup: + enabled: false + +endpoint_config: + endpoints: + - "http://localhost:8000" + api_type: "openai_transcriptions" + api_key: null + +report_dir: logs/whisper_librispeech_offline diff --git a/src/inference_endpoint/async_utils/services/metrics_aggregator/token_metrics.py b/src/inference_endpoint/async_utils/services/metrics_aggregator/token_metrics.py index 36fcf16de..de92d7263 100644 --- a/src/inference_endpoint/async_utils/services/metrics_aggregator/token_metrics.py +++ b/src/inference_endpoint/async_utils/services/metrics_aggregator/token_metrics.py @@ -481,11 +481,21 @@ def _load_tokenizer(self) -> None: except Exception: self._prefix_len = 0 self._baseline = 0 - logger.exception( - "Failed to compute chat-template baseline for %s; tool-call " - "token counts may be over-estimated", - self._tokenizer_name, - ) + if getattr(tok, "chat_template", None) is None: + # A tokenizer with no chat template (e.g. Whisper and other ASR + # tokenizers) has no assistant frame to subtract and never takes + # the messages counting path, so a zero baseline is correct - + # not a failure worth a stack trace on every run. + logger.debug( + "Tokenizer %s has no chat template; baseline is 0", + self._tokenizer_name, + ) + else: + logger.exception( + "Failed to compute chat-template baseline for %s; tool-call " + "token counts may be over-estimated", + self._tokenizer_name, + ) def _setup_shards(self, cores_per_worker: int, n_workers: int) -> None: """Spawn one pinned single-worker process per core block. diff --git a/src/inference_endpoint/commands/probe.py b/src/inference_endpoint/commands/probe.py index 22d5b4d9a..e61a758e0 100644 --- a/src/inference_endpoint/commands/probe.py +++ b/src/inference_endpoint/commands/probe.py @@ -51,7 +51,11 @@ class ProbeConfig(BaseModel): api_type: Annotated[ APIType, cyclopts.Parameter( - alias="--api-type", help="API type: openai, sglang, or videogen" + alias="--api-type", + help="API type: openai, openai_completions, or sglang", + # cyclopts otherwise renders choices from every APIType member, + # including the ones _probe_async rejects below. + show_choices=False, ), ] = APIType.OPENAI requests: int = Field(10, ge=1) @@ -97,6 +101,14 @@ async def _probe_async(config: ProbeConfig) -> None: "(per-request latencies exceed the probe timeout). " "Use a dedicated health check or a benchmark from-config run instead." ) + # The transcriptions body is a multipart audio upload built from a dataset + # row, so the probe's text prompt has nowhere to go. + if api_type == APIType.OPENAI_TRANSCRIPTIONS: + raise InputValidationError( + "Probe does not support api_type=openai_transcriptions " + "(the request body is an audio upload, not a text prompt). " + "Use a dedicated health check or a benchmark from-config run instead." + ) # Model: use provided or default to valid OpenAI model name model_name = config.model diff --git a/src/inference_endpoint/config/rulesets/mlcommons/datasets.py b/src/inference_endpoint/config/rulesets/mlcommons/datasets.py index 58762f94d..e41f4dbaf 100644 --- a/src/inference_endpoint/config/rulesets/mlcommons/datasets.py +++ b/src/inference_endpoint/config/rulesets/mlcommons/datasets.py @@ -56,6 +56,18 @@ def __hash__(self) -> int: size=4388, ) +# speech2text. Size is the "dataset-size" entry for whisper in the inference +# submission checker's constants.py, and matches the repackaged manifest's own +# sample count (dataset_manager/predefined/whisper_librispeech). +LibriSpeechDevAll = _Dataset( + description=( + "LibriSpeech dev-clean + dev-other, converted to 16 kHz WAV and " + "repackaged by the MLCommons pipeline into clips approaching 30 s" + ), + size=1633, + metadata={"scorer": "wer"}, +) + # --- Edge-Agentic (BFCL v4) benchmark datasets --- BFCLv4SingleTurn = _Dataset( diff --git a/src/inference_endpoint/config/rulesets/mlcommons/models.py b/src/inference_endpoint/config/rulesets/mlcommons/models.py index 37618cb5b..e442d654e 100644 --- a/src/inference_endpoint/config/rulesets/mlcommons/models.py +++ b/src/inference_endpoint/config/rulesets/mlcommons/models.py @@ -146,6 +146,14 @@ def __hash__(self) -> int: dataset=datasets.TextGenComplex, ) +Whisper_large_v3 = _Model( + "whisper-large-v3", + golden_accuracy=("bfloat16", {"accuracy": 97.9329}), + accuracy_target_settings=[{"accuracy": (0.99,)}], + dataset=datasets.LibriSpeechDevAll, + use_equal_issue_mode=False, +) + # Edge-Agentic (BFCL v4) reference model. The accuracy gate is BFCL v4 single-turn # (overall + non_live-normalized). golden_accuracy here is the validated Q4_K_M # reference point on a Jetson Thor (llama.cpp, reasoning off); accuracy is diff --git a/src/inference_endpoint/config/rulesets/mlcommons/rules.py b/src/inference_endpoint/config/rulesets/mlcommons/rules.py index 86cfe0c57..e91a169c7 100644 --- a/src/inference_endpoint/config/rulesets/mlcommons/rules.py +++ b/src/inference_endpoint/config/rulesets/mlcommons/rules.py @@ -272,6 +272,13 @@ def apply_user_config( max_ttft_latency_ms=2000, max_tpot_latency_ms=200 ), }, + models.Whisper_large_v3: { + OptimizationPriority.THROUGHPUT: PerModelRuleset( + min_sample_count_valid=1633, + metric=metrics.Throughput, + reported_metrics=[metrics.Throughput], + ), + }, }, ) diff --git a/src/inference_endpoint/config/schema.py b/src/inference_endpoint/config/schema.py index 222d31057..66a1eb066 100644 --- a/src/inference_endpoint/config/schema.py +++ b/src/inference_endpoint/config/schema.py @@ -143,6 +143,7 @@ class ScorerMethod(str, Enum): BFCL_V4 = "bfcl_v4" LEGACY_MLPERF_DEEPSEEK_R1 = "legacy_mlperf_deepseek_r1" SWE_BENCH = "swe_bench_scorer" + WER = "wer" # Audit configuration; runnable test registry lives in compliance/. @@ -1067,7 +1068,8 @@ class EndpointConfig(BaseModel): api_type: Annotated[ APIType, cyclopts.Parameter( - alias="--api-type", help="API type: openai, sglang, or videogen" + alias="--api-type", + help="API type: openai, openai_transcriptions, sglang, or videogen", ), ] = APIType.OPENAI @@ -1169,12 +1171,24 @@ def _resolve_and_validate(self) -> Self: self.benchmark_mode if self.type == TestType.SUBMISSION else self.type ) + # Streaming is a property of the api_type as well as the test type: AUTO + # must not resolve to ON for a protocol with no SSE mode. + api_type = self.endpoint_config.api_type if self.model_params.streaming == StreamingMode.AUTO: mp_updates["streaming"] = ( StreamingMode.OFF - if effective_mode in (TestType.OFFLINE,) + if not api_type.supports_streaming + or effective_mode in (TestType.OFFLINE,) else StreamingMode.ON ) + elif ( + not api_type.supports_streaming + and self.model_params.streaming == StreamingMode.ON + ): + raise ValueError( + f"endpoint_config.api_type={api_type.value} has no streaming " + "mode; set model_params.streaming to 'off' or leave it 'auto'" + ) if not self.model_params.name and self.submission_ref: mp_updates["name"] = self.submission_ref.model @@ -1189,6 +1203,23 @@ def _resolve_and_validate(self) -> Self: if not self.model_params.name: raise ValueError("Required: --model-params.name [--model]") + if not api_type.has_text_prompt: + # No prompt means no ISL to measure, and the aggregator would + # register a series that can only stay empty. Derived rather than + # demanded of the user, since metrics_isl defaults to True. + if self.settings.metrics_isl: + object.__setattr__( + self, + "settings", + self.settings.model_copy(update={"metrics_isl": False}), + ) + if self.settings.warmup.salt: + raise ValueError( + f"settings.warmup.salt is not supported with " + f"endpoint_config.api_type={api_type.value}: the request " + "carries no text prompt to salt" + ) + uses_swe_bench = any( dataset.accuracy_config is not None and dataset.accuracy_config.eval_method == ScorerMethod.SWE_BENCH diff --git a/src/inference_endpoint/config/templates/concurrency_template_full.yaml b/src/inference_endpoint/config/templates/concurrency_template_full.yaml index 7ed842b91..e96238dbc 100644 --- a/src/inference_endpoint/config/templates/concurrency_template_full.yaml +++ b/src/inference_endpoint/config/templates/concurrency_template_full.yaml @@ -43,7 +43,7 @@ datasets: # Dataset configs system: system_prompt generate_params: null # Dataset-specific parameters passed to the generate() method accuracy_config: # Accuracy evaluation settings - eval_method: pass_at_1 # Scorer method | options: pass_at_1, string_match, rouge, code_bench_scorer, shopify_category_f1, agentic_inference_inline, vbench, bfcl_v4, legacy_mlperf_deepseek_r1, swe_bench_scorer + eval_method: pass_at_1 # Scorer method | options: pass_at_1, string_match, rouge, code_bench_scorer, shopify_category_f1, agentic_inference_inline, vbench, bfcl_v4, legacy_mlperf_deepseek_r1, swe_bench_scorer, wer ground_truth: ground_truth # Ground truth column name extractor: boxed_math_extractor # Answer extractor (abcd_extractor, boxed_math_extractor, identity_extractor, python_code_extractor) num_repeats: 1 # Repeat dataset N times for evaluation @@ -112,7 +112,7 @@ endpoint_config: endpoints: # Endpoint URL(s). Must include scheme, e.g. 'http://host:port'. - http://localhost:8000 api_key: null # API key - api_type: openai # API type: openai, sglang, or videogen | options: openai, openai_completions, sglang, videogen + api_type: openai # API type: openai, openai_transcriptions, sglang, or videogen | options: openai, openai_completions, openai_transcriptions, sglang, videogen report_dir: null # Report output directory verbose: false # Enable verbose logging enable_cpu_affinity: true # NUMA-aware CPU pinning diff --git a/src/inference_endpoint/config/templates/offline_template_full.yaml b/src/inference_endpoint/config/templates/offline_template_full.yaml index 6ed1aebb8..a26cdac0a 100644 --- a/src/inference_endpoint/config/templates/offline_template_full.yaml +++ b/src/inference_endpoint/config/templates/offline_template_full.yaml @@ -43,7 +43,7 @@ datasets: # Dataset configs system: system_prompt generate_params: null # Dataset-specific parameters passed to the generate() method accuracy_config: # Accuracy evaluation settings - eval_method: pass_at_1 # Scorer method | options: pass_at_1, string_match, rouge, code_bench_scorer, shopify_category_f1, agentic_inference_inline, vbench, bfcl_v4, legacy_mlperf_deepseek_r1, swe_bench_scorer + eval_method: pass_at_1 # Scorer method | options: pass_at_1, string_match, rouge, code_bench_scorer, shopify_category_f1, agentic_inference_inline, vbench, bfcl_v4, legacy_mlperf_deepseek_r1, swe_bench_scorer, wer ground_truth: ground_truth # Ground truth column name extractor: boxed_math_extractor # Answer extractor (abcd_extractor, boxed_math_extractor, identity_extractor, python_code_extractor) num_repeats: 1 # Repeat dataset N times for evaluation @@ -112,7 +112,7 @@ endpoint_config: endpoints: # Endpoint URL(s). Must include scheme, e.g. 'http://host:port'. - http://localhost:8000 api_key: null # API key - api_type: openai # API type: openai, sglang, or videogen | options: openai, openai_completions, sglang, videogen + api_type: openai # API type: openai, openai_transcriptions, sglang, or videogen | options: openai, openai_completions, openai_transcriptions, sglang, videogen report_dir: null # Report output directory verbose: false # Enable verbose logging enable_cpu_affinity: true # NUMA-aware CPU pinning diff --git a/src/inference_endpoint/config/templates/online_template_full.yaml b/src/inference_endpoint/config/templates/online_template_full.yaml index 36a844354..f46ad9158 100644 --- a/src/inference_endpoint/config/templates/online_template_full.yaml +++ b/src/inference_endpoint/config/templates/online_template_full.yaml @@ -43,7 +43,7 @@ datasets: # Dataset configs system: system_prompt generate_params: null # Dataset-specific parameters passed to the generate() method accuracy_config: # Accuracy evaluation settings - eval_method: pass_at_1 # Scorer method | options: pass_at_1, string_match, rouge, code_bench_scorer, shopify_category_f1, agentic_inference_inline, vbench, bfcl_v4, legacy_mlperf_deepseek_r1, swe_bench_scorer + eval_method: pass_at_1 # Scorer method | options: pass_at_1, string_match, rouge, code_bench_scorer, shopify_category_f1, agentic_inference_inline, vbench, bfcl_v4, legacy_mlperf_deepseek_r1, swe_bench_scorer, wer ground_truth: ground_truth # Ground truth column name extractor: boxed_math_extractor # Answer extractor (abcd_extractor, boxed_math_extractor, identity_extractor, python_code_extractor) num_repeats: 1 # Repeat dataset N times for evaluation @@ -113,7 +113,7 @@ endpoint_config: endpoints: # Endpoint URL(s). Must include scheme, e.g. 'http://host:port'. - http://localhost:8000 api_key: null # API key - api_type: openai # API type: openai, sglang, or videogen | options: openai, openai_completions, sglang, videogen + api_type: openai # API type: openai, openai_transcriptions, sglang, or videogen | options: openai, openai_completions, openai_transcriptions, sglang, videogen report_dir: null # Report output directory verbose: false # Enable verbose logging enable_cpu_affinity: true # NUMA-aware CPU pinning diff --git a/src/inference_endpoint/core/types.py b/src/inference_endpoint/core/types.py index 03cc29942..24f06711c 100644 --- a/src/inference_endpoint/core/types.py +++ b/src/inference_endpoint/core/types.py @@ -36,6 +36,7 @@ class APIType(str, Enum): OPENAI = "openai" OPENAI_COMPLETIONS = "openai_completions" + OPENAI_TRANSCRIPTIONS = "openai_transcriptions" SGLANG = "sglang" VIDEOGEN = "videogen" @@ -46,6 +47,8 @@ def default_route(self) -> str: return "v1/chat/completions" case APIType.OPENAI_COMPLETIONS: return "v1/completions" + case APIType.OPENAI_TRANSCRIPTIONS: + return "v1/audio/transcriptions" case APIType.SGLANG: return "generate" case APIType.VIDEOGEN: @@ -53,6 +56,34 @@ def default_route(self) -> str: case _: raise ValueError(f"Invalid API type: {self}") + @property + def supports_streaming(self) -> bool: + """Whether this API has an SSE mode at all. + + False means streaming is a property of the protocol rather than a user + choice: its adapter's ``decode_sse_message`` raises, and its registered + accumulator is a stub. Config validation resolves AUTO to OFF and + rejects an explicit ON rather than letting the run fail per request. + """ + return self not in _NON_STREAMING_API_TYPES + + @property + def has_text_prompt(self) -> bool: + """Whether a request body carries a text prompt from the dataset row. + + False means anything keyed on prompt text is meaningless here: there is + no input sequence length to tokenize and no prompt to salt. Distinct + from :attr:`supports_streaming` — videogen is non-streaming but is + text-to-video, so it keeps both. + """ + return self not in _NO_TEXT_PROMPT_API_TYPES + + +_NON_STREAMING_API_TYPES = frozenset({APIType.OPENAI_TRANSCRIPTIONS, APIType.VIDEOGEN}) + +#: Audio in, transcript out — no prompt column exists to measure or salt. +_NO_TEXT_PROMPT_API_TYPES = frozenset({APIType.OPENAI_TRANSCRIPTIONS}) + class QueryStatus(Enum): """Status of a query in its lifecycle. diff --git a/src/inference_endpoint/dataset_manager/__init__.py b/src/inference_endpoint/dataset_manager/__init__.py index 98c96f868..ff3db03a0 100644 --- a/src/inference_endpoint/dataset_manager/__init__.py +++ b/src/inference_endpoint/dataset_manager/__init__.py @@ -39,6 +39,7 @@ ShopifyProductCatalogue8k, ) from .predefined.swe_bench import SWEBench +from .predefined.whisper_librispeech import WhisperLibriSpeech from .transforms import ( AddStaticColumns, ColumnFilter, @@ -72,5 +73,6 @@ "ShopifyProductCatalogue", "ShopifyProductCatalogue8k", "SWEBench", + "WhisperLibriSpeech", "AgenticInferenceDataset", ] diff --git a/src/inference_endpoint/dataset_manager/predefined/whisper_librispeech/__init__.py b/src/inference_endpoint/dataset_manager/predefined/whisper_librispeech/__init__.py new file mode 100644 index 000000000..c80198b16 --- /dev/null +++ b/src/inference_endpoint/dataset_manager/predefined/whisper_librispeech/__init__.py @@ -0,0 +1,372 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os +import subprocess +from logging import getLogger +from pathlib import Path, PurePosixPath +from urllib.parse import urlparse + +import pandas as pd + +# `requests` is barred from the benchmark's own HTTP path, which uses the +# httptools-based ConnectionPool in endpoint_client/. This is dataset setup: +# one blocking fetch of the pinned downloader script, run before any load is +# issued and never during a measured request. +import requests + +from ...dataset import Dataset + +logger = getLogger(__name__) + +#: Env var pointing at an already-prepared ``dev-all-repack.json`` manifest, +#: which skips the download. +SOURCE_ENV = "WHISPER_LIBRISPEECH_MANIFEST" + +#: Manifest filename inside the MLCommons-prepared corpus. +MANIFEST_FILENAME = "dev-all-repack.json" + +#: Sample count of the official repackaged dev-all corpus. Logged as a warning +#: on mismatch, not enforced: a truncated manifest is a valid smoke-test input. +MLPERF_SAMPLE_COUNT = 1633 + + +class WhisperLibriSpeech(Dataset, dataset_id="whisper_librispeech"): + """MLPerf speech2text LibriSpeech corpus (MLCommons-prepared manifest). + + Consumes the ``dev-all-repack.json`` manifest and WAV directory produced by + the official MLCommons pipeline, which merges LibriSpeech dev-clean and + dev-other, converts FLAC to 16 kHz WAV, and repackages utterances into + clips approaching 30 s. ``generate()`` downloads it on first use; + ``manifest`` / ``$WHISPER_LIBRISPEECH_MANIFEST`` point at a local copy + instead. + + Building the corpus here is intentionally out of scope. The ~30 s + repackaging is part of the benchmark definition — a client-side + reimplementation would change the clip boundaries and therefore the + reported WER, making the number incomparable to the MLPerf reference. + + Columns: + - ``audio_file`` : absolute path to the clip's WAV, issued by + ``OpenAITranscriptionsAdapter`` (which reads the bytes in the worker). + - ``transcript`` : reference text, read by ``WERScorer`` off the + dataframe as the ground-truth column. + - ``duration`` : clip length in seconds. Retained in the cached parquet + for inspection only — nothing in the benchmark reads it, and the + adapter's ``ColumnFilter`` drops it before issuance. + + One loader serves both phases: the performance phase issues ``audio_file`` + and the accuracy phase scores against ``transcript``. + """ + + COLUMN_NAMES = ["audio_file", "transcript", "duration"] + + #: Published ``.uri`` metadata for the preprocessed speech2text corpus. + #: Override for staging via ``$WHISPER_LIBRISPEECH_DATASET_URI``. + R2_DATASET_URI = ( + "https://inference.mlcommons-storage.org/metadata/whisper-dataset.uri" + ) + R2_DATASET_URI_ENV = "WHISPER_LIBRISPEECH_DATASET_URI" + + #: Pinned mlcommons/r2-downloader commit used by ``_download_from_r2``. + R2_DOWNLOADER_COMMIT = "27da4421877f2831eeb615b43ee5098c4b70be7e" + + #: The resolved dataset URI is handed to a downloaded shell script, so the + #: host is restricted to MLCommons R2 storage. This stops a mis-set + #: ``$WHISPER_LIBRISPEECH_DATASET_URI`` from pointing the downloader at an + #: attacker-controlled location. + R2_ALLOWED_HOST = "mlcommons-storage.org" + + #: Wall-clock ceiling for the downloader invocation. Dataset generation runs + #: in ``setup_benchmark``, before ``RunWatchdog`` arms its whole-run + #: deadline, so nothing else would ever interrupt a stalled fetch. Sized for + #: several GB of audio over a slow link, not for a healthy transfer. + R2_DOWNLOAD_TIMEOUT_S = 2 * 60 * 60 + + @classmethod + def _download_from_r2(cls, uri: str, download_dir: Path) -> None: + """Download the MLCommons-hosted speech2text corpus into ``download_dir``. + + Fetches the pinned ``mlc-r2-downloader.sh`` and runs it against the + dataset's ``.uri`` metadata. The script MD5-verifies every file it pulls + and fails its exit code on mismatch, so the caller only needs to locate + the manifest afterwards. + + Args: + uri: The dataset's published ``.uri`` metadata URL. + download_dir: Directory the corpus is downloaded into. + + Raises: + ValueError: ``uri`` is not https on MLCommons R2 storage. + RuntimeError: the downloader exited non-zero or outran + ``R2_DOWNLOAD_TIMEOUT_S``. + """ + parsed = urlparse(uri) + host = parsed.hostname or "" + if parsed.scheme != "https" or not ( + host == cls.R2_ALLOWED_HOST or host.endswith("." + cls.R2_ALLOWED_HOST) + ): + raise ValueError( + f"Refusing to download dataset from untrusted URI '{uri}': " + f"expected https on {cls.R2_ALLOWED_HOST}" + ) + + download_dir.mkdir(parents=True, exist_ok=True) + script_path = download_dir / "mlc-r2-downloader.sh" + resp = requests.get( + "https://raw.githubusercontent.com/mlcommons/r2-downloader/" + f"{cls.R2_DOWNLOADER_COMMIT}/mlc-r2-downloader.sh", + timeout=30, + ) + resp.raise_for_status() + script_path.write_bytes(resp.content) + script_path.chmod(0o755) + + logger.info("Downloading %s into %s", uri, download_dir) + try: + result = subprocess.run( + ["bash", str(script_path.resolve()), "-d", str(download_dir), uri], + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + text=True, + check=False, + timeout=cls.R2_DOWNLOAD_TIMEOUT_S, + ) + except subprocess.TimeoutExpired as e: + raise RuntimeError( + f"R2 downloader for '{uri}' exceeded {cls.R2_DOWNLOAD_TIMEOUT_S}s " + f"and was killed; whatever it had written is left under " + f"{download_dir}. Raise the timeout, or point generate_params at " + "a local copy of the corpus." + ) from e + finally: + script_path.unlink(missing_ok=True) + + if result.returncode != 0: + raise RuntimeError( + f"R2 downloader failed with code {result.returncode}: {result.stderr}" + ) + + @classmethod + def generate( + cls, + datasets_dir: Path, + manifest: str | os.PathLike | None = None, + data_dir: str | os.PathLike | None = None, + max_samples: int | None = None, + seed: int = 0, + force: bool = False, + ) -> pd.DataFrame: + """Build (or load the cached) LibriSpeech benchmark dataframe. + + Args: + datasets_dir: Root cache dir; the parquet is written under + ``/whisper_librispeech/`` and the downloaded + corpus lands in that directory's ``source/`` subdirectory. + manifest: An already-prepared ``dev-all-repack.json``, which skips + the download. Falls back to ``$WHISPER_LIBRISPEECH_MANIFEST``, + then to the MLCommons R2 download. + data_dir: Root the manifest's ``fname`` entries are relative to. + Defaults to the manifest's parent directory. Leading ``fname`` + components that are absent under the root are dropped, so the + hosted artifact's flattened layout resolves without an override. + max_samples: If set, keep the first N rows. Manifest order is the + MLPerf sample order, so truncating (rather than sampling) keeps + a smoke run's indices aligned with the full corpus. + seed: Unused; present for the ``get_dataloader`` contract. + force: Rebuild the cached parquet even if it already exists. An + already-downloaded corpus is reused — this does not re-fetch + several GB of audio. The cache path is keyed on the dataset ID + alone, so changing ``manifest`` or ``data_dir`` reuses the rows + built from the previous one until this is set. + """ + dst_path = datasets_dir / cls.DATASET_ID / "whisper_librispeech.parquet" + dst_path.parent.mkdir(parents=True, exist_ok=True) + + if dst_path.exists() and not force: + logger.info("LibriSpeech dataset cached at %s; loading.", dst_path) + df = pd.read_parquet(dst_path) + else: + manifest_path = cls._resolve_manifest(manifest, dst_path.parent / "source") + df = cls._load_manifest(manifest_path, data_dir) + df.to_parquet(dst_path, index=False) + logger.info("Wrote %d LibriSpeech rows to %s", len(df), dst_path) + + if len(df) != MLPERF_SAMPLE_COUNT: + logger.warning( + "Manifest has %d samples; the official MLPerf speech2text " + "corpus has %d. A submission run must use the full corpus.", + len(df), + MLPERF_SAMPLE_COUNT, + ) + + if max_samples is not None and max_samples < len(df): + logger.info("Truncating to the first %d of %d rows", max_samples, len(df)) + df = df.head(max_samples).reset_index(drop=True) + return df + + @classmethod + def _resolve_manifest( + cls, + manifest: str | os.PathLike | None, + download_dir: Path, + ) -> Path: + """Locate the manifest, downloading the corpus if nothing local is given. + + Precedence: explicit ``manifest`` / ``$WHISPER_LIBRISPEECH_MANIFEST``, + then an earlier download under ``download_dir``, then a fresh R2 fetch. + """ + resolved = manifest or os.environ.get(SOURCE_ENV) + if resolved: + manifest_path = Path(resolved) + if not manifest_path.exists(): + raise FileNotFoundError( + f"LibriSpeech manifest not found at {manifest_path}" + ) + if manifest_path.suffix != ".json": + raise NotImplementedError( + f"LibriSpeech source {manifest_path} is not a .json manifest. " + f"Only the MLCommons-prepared {MANIFEST_FILENAME} is supported; " + "building the corpus from raw LibriSpeech is out of scope." + ) + return manifest_path + + cached = cls._find_manifest(download_dir) + if cached is not None: + logger.info("Using the downloaded LibriSpeech corpus at %s", cached) + return cached + + logger.info( + "Downloading the MLCommons speech2text corpus (several GB of audio) " + "to %s; pass generate_params.manifest to use a local copy instead.", + download_dir, + ) + cls._download_from_r2( + os.environ.get(cls.R2_DATASET_URI_ENV) or cls.R2_DATASET_URI, download_dir + ) + found = cls._find_manifest(download_dir) + if found is None: + raise FileNotFoundError( + f"R2 download completed but no {MANIFEST_FILENAME} was found " + f"under {download_dir}; the hosted artifact must carry the " + "repackaged manifest." + ) + return found + + @staticmethod + def _find_manifest(download_dir: Path) -> Path | None: + """Find the manifest anywhere under a downloaded corpus, or None. + + The WAV directory sits beside it, so the caller's default ``data_dir`` + (the manifest's parent) is the right root; ``_fname_strip`` absorbs the + relative prefix the artifact's layout does not reproduce. + """ + if not download_dir.exists(): + return None + return next(download_dir.rglob(MANIFEST_FILENAME), None) + + @classmethod + def _load_manifest( + cls, + manifest_path: Path, + data_dir: str | os.PathLike | None, + ) -> pd.DataFrame: + root = Path(data_dir) if data_dir is not None else manifest_path.parent + entries = json.loads(manifest_path.read_text()) + clips = [cls._clip(entry, i) for i, entry in enumerate(entries)] + strip = cls._fname_strip(root, clips[0]["fname"]) if clips else 0 + rows = [ + cls._row(entry, clip, root, strip) + for entry, clip in zip(entries, clips, strict=True) + ] + return pd.DataFrame(rows, columns=cls.COLUMN_NAMES) + + @staticmethod + def _fname_strip(root: Path, fname: str) -> int: + """Leading ``fname`` components to drop so the clip resolves under root. + + The manifest's ``fname`` entries are written relative to the upstream + pipeline's working directory (``./data/dev-all-repack/.wav``), but + the hosted artifact unpacks the WAV directory straight into the corpus + root, so a literal join misses by one component. Probe the first clip + against each suffix of its relative path and reuse the offset for the + whole manifest: the manifest is internally consistent, so one lookup + settles the layout. + + Falls back to 0 (a literal join) when nothing resolves, so a manifest + describing audio that is not mounted here still produces a dataframe. + """ + parts = PurePosixPath(fname).parts + for strip in range(len(parts)): + if root.joinpath(*parts[strip:]).exists(): + if strip: + logger.info( + "Manifest fname prefix %s is not present under %s; " + "resolving clips as %s", + "/".join(parts[:strip]), + root, + root.joinpath(*parts[strip:]), + ) + return strip + logger.warning( + "No audio found for the first manifest clip %s under %s. Clip paths " + "are recorded as-is and will fail at request time; pass " + "generate_params.data_dir if the audio lives elsewhere.", + fname, + root, + ) + return 0 + + @staticmethod + def _clip(entry: dict, index: int) -> dict: + """Select one manifest entry's audio clip. + + The upstream schema nests the clip under ``files`` (a list, to carry + speed-perturbed variants). Like the MLPerf reference's manifest reader, + pick the unperturbed ``speed == 1`` clip; the repackaged dev-all + manifest carries exactly one such entry per sample. + """ + files = entry.get("files") + if not files: + raise ValueError(f"Manifest entry {index} has no 'files' list: {entry}") + if "transcript" not in entry: + raise ValueError( + f"Manifest entry {index} has no inline 'transcript'. The " + "repackaged dev-all manifest carries transcripts inline; a " + "'text_filepath' sidecar manifest is not supported." + ) + clip = next((f for f in files if f.get("speed", 1) == 1), None) + if clip is None: + raise ValueError( + f"Manifest entry {index} has no unperturbed (speed == 1) clip; " + "speed-perturbed-only manifests are not supported." + ) + return clip + + @staticmethod + def _row(entry: dict, clip: dict, root: Path, strip: int) -> dict: + """Flatten one manifest entry into a benchmark row.""" + parts = PurePosixPath(clip["fname"]).parts + return { + "audio_file": str(root.joinpath(*parts[strip:])), + "transcript": entry["transcript"], + "duration": float( + entry.get("original_duration", clip.get("duration", 0.0)) + ), + } + + +__all__ = ["WhisperLibriSpeech"] diff --git a/src/inference_endpoint/endpoint_client/accumulator_protocol.py b/src/inference_endpoint/endpoint_client/accumulator_protocol.py index 5aea629e9..92305bdc7 100644 --- a/src/inference_endpoint/endpoint_client/accumulator_protocol.py +++ b/src/inference_endpoint/endpoint_client/accumulator_protocol.py @@ -70,3 +70,35 @@ def get_final_output(self) -> QueryResult: # pragma: no cover QueryResult with the complete response output. """ ... + + +class NonStreamingAccumulator: + """Accumulator stub for api_types whose endpoint has no SSE mode. + + HTTPClientConfig resolves an adapter/accumulator pair for every api_type, so + a non-streaming endpoint still needs an accumulator. This one should never be + exercised. + + `get_final_output` raises rather than returning an empty `QueryResult` + because a streamed response body carrying no `data:` frames never calls + `decode_sse_message` at all: `parse_sse_chunk` finds no matches, nothing + raises, and the stream ends with nothing accumulated. Returning an empty + result there would record a zero-output query as successful. A body that + does carry frames raises `NotImplementedError` out of the adapter instead — + `parse_sse_chunk` catches only msgspec decode errors — and the worker + records that query as failed. + """ + + def __init__(self, query_id: str, stream_all_chunks: bool) -> None: + self.query_id = query_id + # stream_all_chunks is intentionally ignored: non-streaming endpoint. + + def add_chunk(self, delta: Any) -> StreamChunk | None: + return None + + def get_final_output(self) -> QueryResult: + raise RuntimeError( + "NonStreamingAccumulator.get_final_output called for query " + f"{self.query_id}: this api_type has no streaming mode - check " + "HTTPClientConfig.streaming and query.data['stream']." + ) diff --git a/src/inference_endpoint/endpoint_client/adapter_protocol.py b/src/inference_endpoint/endpoint_client/adapter_protocol.py index a96faaa0a..92ae4ba24 100644 --- a/src/inference_endpoint/endpoint_client/adapter_protocol.py +++ b/src/inference_endpoint/endpoint_client/adapter_protocol.py @@ -20,7 +20,7 @@ import logging import re from abc import ABC, abstractmethod -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, ClassVar import msgspec @@ -46,6 +46,12 @@ class HttpRequestAdapter(ABC): # Matches "data: {json content}" and captures the JSON part SSE_DATA_PATTERN: re.Pattern[bytes] = re.compile(rb"data:\s*(\{[^\n]+\})") + # Media type of the bytes returned by encode_query(). The worker folds this + # into the pre-encoded request prefix once at setup, so adapters that send + # something other than JSON (e.g. a multipart audio upload) declare it here + # rather than emitting a per-request header. + CONTENT_TYPE: ClassVar[str] = "application/json" + @classmethod @abstractmethod def dataset_transforms( diff --git a/src/inference_endpoint/endpoint_client/config.py b/src/inference_endpoint/endpoint_client/config.py index b2839996d..8146d5f15 100644 --- a/src/inference_endpoint/endpoint_client/config.py +++ b/src/inference_endpoint/endpoint_client/config.py @@ -64,6 +64,7 @@ def _endpoint_destination(url: str) -> tuple[str | None, int]: ADAPTER_MAP = { APIType.OPENAI: "inference_endpoint.openai.openai_msgspec_adapter.OpenAIMsgspecAdapter", APIType.OPENAI_COMPLETIONS: "inference_endpoint.openai.completions_adapter.OpenAITextCompletionsAdapter", + APIType.OPENAI_TRANSCRIPTIONS: "inference_endpoint.openai.transcriptions_adapter.OpenAITranscriptionsAdapter", APIType.SGLANG: "inference_endpoint.sglang.adapter.SGLangGenerateAdapter", APIType.VIDEOGEN: "inference_endpoint.videogen.adapter.VideoGenAdapter", } @@ -71,6 +72,7 @@ def _endpoint_destination(url: str) -> tuple[str | None, int]: ACCUMULATOR_MAP = { APIType.OPENAI: "inference_endpoint.openai.accumulator.OpenAISSEAccumulator", APIType.OPENAI_COMPLETIONS: "inference_endpoint.openai.accumulator.OpenAISSEAccumulator", + APIType.OPENAI_TRANSCRIPTIONS: "inference_endpoint.endpoint_client.accumulator_protocol.NonStreamingAccumulator", APIType.SGLANG: "inference_endpoint.sglang.accumulator.SGLangSSEAccumulator", APIType.VIDEOGEN: "inference_endpoint.videogen.adapter.VideoGenAccumulator", } diff --git a/src/inference_endpoint/endpoint_client/http.py b/src/inference_endpoint/endpoint_client/http.py index 6d70f93ab..706e345a3 100644 --- a/src/inference_endpoint/endpoint_client/http.py +++ b/src/inference_endpoint/endpoint_client/http.py @@ -699,30 +699,35 @@ class HttpRequestTemplate: Attributes: static_prefix: Pre-merged request line + host header bytes. cached_headers: Pre-encoded headers from cache_headers(), included in every request. + content_type: Request body media type, declared by the adapter. """ __slots__ = ( "static_prefix", "cached_headers", + "content_type", "_prefix_streaming", "_prefix_non_streaming", ) - # Pre-encoded general headers - HEADERS_STREAMING = ( - b"Content-Type: application/json\r\nAccept: text/event-stream\r\n" - ) - HEADERS_NON_STREAMING = ( - b"Content-Type: application/json\r\nAccept: application/json\r\n" - ) + DEFAULT_CONTENT_TYPE = "application/json" - def __init__(self, static_prefix: bytes): + def __init__( + self, static_prefix: bytes, content_type: str = DEFAULT_CONTENT_TYPE + ) -> None: self.static_prefix = static_prefix self.cached_headers = b"" + self.content_type = content_type self._rebuild_prefixes() @classmethod - def from_url(cls, host: str, port: int, path: str = "/") -> HttpRequestTemplate: + def from_url( + cls, + host: str, + port: int, + path: str = "/", + content_type: str = DEFAULT_CONTENT_TYPE, + ) -> HttpRequestTemplate: """ Create an HttpRequestTemplate from URL components. @@ -730,6 +735,8 @@ def from_url(cls, host: str, port: int, path: str = "/") -> HttpRequestTemplate: host: Target hostname port: Target port path: Request path (e.g., "/v1/chat/completions") + content_type: Request body media type (e.g. "multipart/form-data; + boundary=..." for an audio upload) Returns: HttpRequestTemplate ready for building requests @@ -745,13 +752,19 @@ def from_url(cls, host: str, port: int, path: str = "/") -> HttpRequestTemplate: else: host_header = f"Host: {host}:{port}\r\n" - return cls(static_prefix=(request_line + host_header).encode("ascii")) + return cls( + static_prefix=(request_line + host_header).encode("ascii"), + content_type=content_type, + ) def _rebuild_prefixes(self) -> None: """Merge static_prefix + cached_headers + content-type into two ready-to-use prefixes.""" base = self.static_prefix + self.cached_headers - self._prefix_streaming = base + self.HEADERS_STREAMING - self._prefix_non_streaming = base + self.HEADERS_NON_STREAMING + content_type = f"Content-Type: {self.content_type}\r\n".encode("ascii") + self._prefix_streaming = base + content_type + b"Accept: text/event-stream\r\n" + self._prefix_non_streaming = ( + base + content_type + b"Accept: application/json\r\n" + ) def cache_headers(self, headers: dict[str, str]) -> None: """ @@ -782,7 +795,7 @@ def build_request( Build a complete HTTP/1.1 request as raw bytes. Args: - body: Request body bytes (JSON payload) + body: Request body bytes, already encoded in `content_type` streaming: True for SSE streaming, False for buffered response extra_headers: Optional additional headers (e.g., Authorization) diff --git a/src/inference_endpoint/endpoint_client/worker.py b/src/inference_endpoint/endpoint_client/worker.py index 321528baa..51335e357 100644 --- a/src/inference_endpoint/endpoint_client/worker.py +++ b/src/inference_endpoint/endpoint_client/worker.py @@ -198,7 +198,10 @@ async def run(self) -> None: # Initialize HTTP template from URL components self._http_template = HttpRequestTemplate.from_url( - self._host, self._port, self._path + self._host, + self._port, + self._path, + content_type=self._adapter.CONTENT_TYPE, ) if self.http_config.api_key: self._http_template.cache_headers( diff --git a/src/inference_endpoint/evaluation/scoring.py b/src/inference_endpoint/evaluation/scoring.py index 61bec6575..fe0efee55 100644 --- a/src/inference_endpoint/evaluation/scoring.py +++ b/src/inference_endpoint/evaluation/scoring.py @@ -36,6 +36,8 @@ import pandas as pd from pydantic import ValidationError from tqdm import tqdm +from transformers import AutoTokenizer +from transformers.models.whisper.english_normalizer import EnglishTextNormalizer try: import websocket @@ -54,11 +56,13 @@ from ..dataset_manager.agentic_inference_dataset import AgenticInferenceDataset from ..dataset_manager.dataset import Dataset from ..dataset_manager.predefined.shopify_product_catalogue import ProductMetadata +from ..exceptions import InputValidationError from .accuracy_results import build_breakdown from .extractor import ( Extractor, PythonCodeExtractor, ) +from .wer import project_to_labels, wer_counts logger = logging.getLogger(__name__) @@ -492,6 +496,150 @@ def score(self) -> tuple[float, int]: return result, 1 +class WERScorer(Scorer, scorer_id="wer"): + """Word Error Rate as defined by the MLPerf speech2text benchmark. + + Reports ``100 - WER%`` so the headline number is higher-is-better and on the + percentage scale the compliance accuracy gate assumes. The MLPerf reference + is WER 2.0671% (accuracy 97.9329%); a submission needs 99% of that. + + Scoring order matches the reference exactly: project both sides through the + character-label inventory, apply Whisper's ``EnglishTextNormalizer``, split + on whitespace, then accumulate the concatenation-aware edit distance. See + :mod:`inference_endpoint.evaluation.wer`. + + An issued sample with no COMPLETE event is scored as an empty hypothesis: + its reference words enter the denominator and every one of them counts as a + deletion, exactly as a completed-but-blank response already does. Skipping + it instead would shrink the denominator, so dropping the samples a SUT + handles worst would *raise* the score. This matches the reference number + whenever nothing is missing, and ``complete`` is set False when something + is, so a partial run is machine-detectable rather than needing a reader to + notice ``missing_samples``. + """ + + REQUIRES_EXTRACTOR = False + + #: Repo (or local dir) supplying ``normalizer.json`` for the English text + #: normalizer. Whisper ships the same mapping across model sizes. The HF + #: copy is a superset of ``whisper/normalizers/english.json`` by one key + #: (``mm``) with no conflicting values, and `transformers`' + #: ``EnglishTextNormalizer`` is output-identical to the one in the + #: ``openai-whisper`` package. + DEFAULT_NORMALIZER_SOURCE = "openai/whisper-large-v3" + + def __init__(self, *args, normalizer_source: str | None = None, **kwargs): + super().__init__(*args, **kwargs) + if self.extractor is not None: + raise InputValidationError( + f"accuracy_config.extractor is not supported by " + f"{type(self).__name__}: WER scores the transcript verbatim, so " + "an extractor would be silently ignored. Remove it." + ) + self.normalizer_source = normalizer_source or self.DEFAULT_NORMALIZER_SOURCE + self._breakdown: dict[str, Any] | None = None + + @classmethod + def preflight(cls, extras: dict[str, Any]) -> None: + """Resolve the normalizer before the run, not after it. + + ``_normalizer`` needs network access or a warm HF cache. Scoring happens + after the whole performance phase, so a cold cache in an air-gapped + submission environment would otherwise fail the run at its very last + step, with the endpoint work already spent. + """ + cls._load_normalizer(extras.get("normalizer_source")) + + @classmethod + def _load_normalizer(cls, source: str | None) -> EnglishTextNormalizer: + resolved = source or cls.DEFAULT_NORMALIZER_SOURCE + tokenizer = AutoTokenizer.from_pretrained(resolved) + spelling_map = getattr(tokenizer, "english_spelling_normalizer", None) + if not spelling_map: + raise ValueError( + f"Tokenizer {resolved!r} carries no " + "english_spelling_normalizer (normalizer.json). Point " + "accuracy_config.extras.normalizer_source at a Whisper " + "tokenizer repo or local directory." + ) + return EnglishTextNormalizer(spelling_map) + + def _normalizer(self) -> EnglishTextNormalizer: + return self._load_normalizer(self.normalizer_source) + + def score_single_sample(self, value: str, ground_truth: str) -> float: + raise RuntimeError( + "WER is a corpus-level metric: errors and reference words are summed " + "across the dataset before dividing. Call score() instead." + ) + + def score(self) -> tuple[float | None, int]: + df = self.get_scoring_outputs() + df = df[df["sample_uuid"].isin(self.sample_index_map.keys())] + + issued = len(self.sample_index_map) + if issued == 0: + self.complete = False + return None, 0 + n_repeats = issued // self.dataset.num_samples() + + assert ( + self.dataset.dataframe is not None + ), f"Dataset {self.dataset} has no dataframe loaded" + assert ( + self.ground_truth_column in self.dataset.dataframe.columns + ), f"Ground truth column {self.ground_truth_column} not found in dataset {self.dataset}" + + normalizer = self._normalizer() + references = self.dataset.dataframe[self.ground_truth_column].to_numpy() + outputs = dict(zip(df["sample_uuid"], df["output"], strict=True)) + + # Iterate the issued samples, not the returned ones: a sample with no + # COMPLETE event is scored as an empty hypothesis (all of its reference + # words count as deletions) rather than dropped from the denominator. + distance = 0 + words = 0 + empty = 0 + missing = 0 + for sample_uuid, index in self.sample_index_map.items(): + hypothesis = outputs.get(sample_uuid) + if hypothesis is None: + missing += 1 + hypothesis = "" + elif not hypothesis: + empty += 1 + hyp_words = normalizer(project_to_labels(hypothesis)).split() + ref_words = normalizer(project_to_labels(str(references[index]))).split() + sample_distance, sample_words = wer_counts(hyp_words, ref_words) + distance += sample_distance + words += sample_words + + self.complete = missing == 0 + if words == 0: + self._breakdown = {"wer_percent": None, "missing_samples": missing} + return None, n_repeats + + wer = distance / words + self._breakdown = { + "wer_percent": wer * 100.0, + "edit_distance": distance, + "reference_words": words, + "empty_responses": empty, + "missing_samples": missing, + } + if missing: + logger.warning( + "%d of %d issued samples produced no output and were scored as " + "empty transcripts; this run is reported as incomplete.", + missing, + issued, + ) + return 100.0 - wer * 100.0, n_repeats + + def score_breakdown(self) -> dict[str, Any] | None: + return self._breakdown + + def _uv_subproject_env(project_path: os.PathLike | str) -> dict[str, str]: """Env for ``uv run --project `` pinned to the subproject's OWN ``.venv``. diff --git a/src/inference_endpoint/evaluation/wer.py b/src/inference_endpoint/evaluation/wer.py new file mode 100644 index 000000000..f88792ef7 --- /dev/null +++ b/src/inference_endpoint/evaluation/wer.py @@ -0,0 +1,158 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Word Error Rate as defined by the MLPerf speech2text benchmark. + +Ported from the MLCommons reference implementation +(https://github.com/mlcommons/inference/tree/master/speech2text), which is also +Apache-2.0: ``legacy_helpers.__levenshtein``, +``helpers.compute_wer_with_concatenation``, and the character-label projection +``accuracy_eval.py`` applies to both sides before scoring. + +The concatenation-aware alignment is why a generic WER library (jiwer, +torchmetrics) will not reproduce the reference number: the reference credits a +hypothesis word that equals up to three joined reference words, and vice versa, +before measuring edit distance. See ``docs/evaluation/DESIGN.md``. + +Parity with upstream is pinned by the golden vectors in +``tests/unit/evaluation/test_wer.py`` (``UPSTREAM_VECTORS``), captured by running +the reference functions at mlcommons/inference commit +``3fbc329939999c13d0a7b5e67fb2092287e06047``. + +Upstream reports ``accuracy = (1 - wer) * 100`` (``speech2text/accuracy_eval.py``), +which is the ``100 - WER%`` scale :class:`WERScorer` reports on. +""" + +from collections.abc import Sequence + +#: Character inventory the reference projects both hypotheses and references +#: through before normalization. Anything outside it is dropped. +LABELS = frozenset(" abcdefghijklmnopqrstuvwxyz'0123456789$%+-") + +#: Longest run of words the alignment will try to join on either side. +MAX_CONCATENATION_SPAN = 3 + + +def project_to_labels(text: str) -> str: + """Lowercase `text` and drop every character outside :data:`LABELS`. + + The reference encodes both the SUT output and the manifest transcript as + indices into its label list and decodes them back before scoring, which is + lossy in exactly this way. Reproducing it keeps punctuation handling + identical to the reference. + + The lowercasing is applied to both sides, which is a no-op on the official + corpus: the MLCommons preparation pipeline already lowercases every + transcript (``speech2text/utils/preprocessing_utils.py`` writes + ``data['transcript'].lower().strip()``), so the manifest this loader + consumes is lowercase before it arrives. + + It is kept as defence for a hand-built manifest. Upstream's + ``manifest.parse_transcript`` maps each character through the label + inventory and *drops* whatever is absent, and the inventory holds no + uppercase letters — so an uppercase transcript would reduce to an empty + reference there rather than failing loudly. Lowercasing first is the only + behaviour that still yields a meaningful number in that case. + """ + lowered = text.lower() + return "".join(c for c in lowered if c in LABELS) + + +def levenshtein(a: Sequence[str], b: Sequence[str]) -> int: + """Edit distance between two word sequences, in O(min(len(a), len(b))) space.""" + n, m = len(a), len(b) + if n > m: + a, b = b, a + n, m = m, n + + current = list(range(n + 1)) + for i in range(1, m + 1): + previous, current = current, [i] + [0] * n + for j in range(1, n + 1): + add, delete = previous[j] + 1, current[j - 1] + 1 + change = previous[j - 1] + if a[j - 1] != b[i - 1]: + change += 1 + current[j] = min(add, delete, change) + return current[n] + + +def wer_counts(hypothesis: Sequence[str], reference: Sequence[str]) -> tuple[int, int]: + """Return ``(edit_distance, reference_word_count)`` for one sample. + + Walks both word lists in lockstep, consuming a joined run of up to + :data:`MAX_CONCATENATION_SPAN` words on either side when it matches a single + word on the other, then takes the edit distance over the reconciled + sequences. Corpus WER is ``sum(distances) / sum(word_counts)`` — never the + mean of per-sample rates. + """ + alignment: list[tuple[str | None, str | None]] = [] + i = j = 0 + + while i < len(reference) and j < len(hypothesis): + if reference[i] == hypothesis[j]: + alignment.append((reference[i], hypothesis[j])) + i += 1 + j += 1 + continue + + ref_match_len = hyp_match_len = 1 + match_found = False + # From 2: a k of 1 would re-test reference[i] == hypothesis[j], which the + # branch above already handled and left unequal. + for k in range(2, MAX_CONCATENATION_SPAN + 1): + if ( + i + k <= len(reference) + and "".join(reference[i : i + k]) == hypothesis[j] + ): + ref_match_len, hyp_match_len = k, 1 + match_found = True + break + if ( + j + k <= len(hypothesis) + and "".join(hypothesis[j : j + k]) == reference[i] + ): + ref_match_len, hyp_match_len = 1, k + match_found = True + break + + if match_found: + alignment.append( + ( + " ".join(reference[i : i + ref_match_len]), + " ".join(hypothesis[j : j + hyp_match_len]), + ) + ) + i += ref_match_len + j += hyp_match_len + else: + # No reconciliation possible: count it as a substitution and advance + # both sides, matching the reference's alignment walk. + alignment.append((reference[i], hypothesis[j])) + i += 1 + j += 1 + + while i < len(reference): + alignment.append((reference[i], None)) + i += 1 + while j < len(hypothesis): + alignment.append((None, hypothesis[j])) + j += 1 + + # Spaces are stripped so a reconciled pair ("can not", "cannot") compares + # equal; the distance is measured over these joined tokens. + ref_aligned = [r.replace(" ", "") for r, _ in alignment if r is not None] + hyp_aligned = [h.replace(" ", "") for _, h in alignment if h is not None] + return levenshtein(ref_aligned, hyp_aligned), len(reference) diff --git a/src/inference_endpoint/openai/transcriptions_adapter.py b/src/inference_endpoint/openai/transcriptions_adapter.py new file mode 100644 index 000000000..5630cb7e2 --- /dev/null +++ b/src/inference_endpoint/openai/transcriptions_adapter.py @@ -0,0 +1,203 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""OpenAI /v1/audio/transcriptions adapter for Whisper-class ASR endpoints.""" + +from pathlib import Path +from typing import ClassVar + +import msgspec +from inference_endpoint.config.schema import ModelParams +from inference_endpoint.core.types import Query, QueryResult, TextModelOutput +from inference_endpoint.dataset_manager.transforms import ( + AddStaticColumns, + ColumnFilter, + Transform, +) +from inference_endpoint.endpoint_client.adapter_protocol import HttpRequestAdapter + +from .types import TranscriptionResponse + +#: Dataset column holding the path to each sample's audio file. +AUDIO_FILE_COLUMN = "audio_file" + +#: The worker folds Content-Type into its pre-encoded request prefix once at +#: setup, so the boundary cannot vary per request and must be a constant. It is +#: long and non-ASCII-audio-like enough that a collision with WAV payload bytes +#: is not a practical concern. +_BOUNDARY = "----mlperf-inference-endpoints-audio-boundary-Q3nV7xTt" + +_DELIMITER = f"--{_BOUNDARY}" +_DASHES = f"{_DELIMITER}\r\n".encode("ascii") +_CLOSING = f"{_DELIMITER}--\r\n".encode("ascii") + +#: Characters that would terminate a `Content-Disposition` parameter early, with +#: the percent escapes browsers substitute for them (WHATWG HTML, "multipart/ +#: form-data encoding algorithm"). Servers decode these, so escaping keeps the +#: original value recoverable where rejecting the row would not. +_PARAM_ESCAPES = str.maketrans({'"': "%22", "\r": "%0D", "\n": "%0A"}) + + +def _escape_param(value: str) -> str: + """Escape a quoted-string header parameter (a `name=` or `filename=` value).""" + return value.translate(_PARAM_ESCAPES) + + +def encode_multipart( + fields: dict[str, str], filename: str, audio: bytes, mime_type: str +) -> bytes: + """Encode `fields` plus one `file` part as an RFC 7578 multipart body. + + Both the parameter names and the filename are dataset-controlled, so they are + escaped: an unescaped `"` or CRLF would close the parameter early and let a + row inject its own headers or parts. Field *values* sit in a part's body + where CRLF is legal, so they are checked only for the one string that would + forge a part boundary. + + Args: + fields: Text form fields, in the order they should be written. + filename: Value of the file part's `filename` parameter. + audio: Raw audio bytes. + mime_type: Media type of the file part. + + Raises: + ValueError: A field value contains the multipart delimiter. + """ + parts: list[bytes] = [] + for name, value in fields.items(): + if _DELIMITER in value: + raise ValueError( + f"Form field {name!r} contains the multipart delimiter " + f"{_DELIMITER!r}, which would forge a part boundary." + ) + parts.append(_DASHES) + parts.append( + f'Content-Disposition: form-data; name="{_escape_param(name)}"' + f"\r\n\r\n{value}\r\n".encode() + ) + parts.append(_DASHES) + parts.append( + f'Content-Disposition: form-data; name="file"; ' + f'filename="{_escape_param(filename)}"\r\n' + f"Content-Type: {mime_type}\r\n\r\n".encode() + ) + parts.append(audio) + parts.append(b"\r\n") + parts.append(_CLOSING) + return b"".join(parts) + + +class OpenAITranscriptionsAdapter(HttpRequestAdapter): + """Adapter for OpenAI /v1/audio/transcriptions (e.g. vLLM serving Whisper). + + The request is a `multipart/form-data` audio upload, so the body is not JSON + and `CONTENT_TYPE` carries the boundary. The dataset row holds a *path*; the + bytes are read here, inside the worker, so a full audio corpus never + traverses the issue thread or the ZMQ frame. + + The read is synchronous on the worker's event loop, so it stalls every + in-flight response in that worker process, not just this request. Keep the + audio on fast local storage: a cold-cache or network-mounted read shows up + as added latency across the whole worker. + + `language` is what makes the server build Whisper's + `<|startoftranscript|><|LANG|><|transcribe|><|notimestamps|>` prompt instead + of auto-detecting per clip, which is how this matches the MLPerf speech2text + reference. `DEFAULT_LANGUAGE` supplies it; a dataset `language` column wins. + + The endpoint is non-streaming: OpenAI does not support `stream` for + Whisper-class models, so `decode_sse_message` raises and the registered + accumulator is `NonStreamingAccumulator`. Config validation rejects + `streaming: on` for this api_type, and `ColumnFilter` admits no `stream` + column, so `query.data` never carries one. + """ + + CONTENT_TYPE: ClassVar[str] = f"multipart/form-data; boundary={_BOUNDARY}" + + #: Form fields forwarded from the dataset row, in request order. `model` + #: and `file` are required by the API; the rest are optional and omitted + #: when absent or None so server-side defaults apply. + FORM_FIELDS: ClassVar[tuple[str, ...]] = ( + "model", + "language", + "prompt", + "response_format", + "temperature", + ) + + #: WAV is the only format the MLPerf LibriSpeech corpus ships. Servers + #: sniff the payload rather than trusting this part header (vLLM hands the + #: bytes to librosa/soundfile), so a per-row MIME column buys nothing. + MIME_TYPE: ClassVar[str] = "audio/wav" + + #: Transcription language, sent on every request. The MLPerf speech2text + #: corpus is English LibriSpeech and the reference decodes with the language + #: pinned, so leaving the server to auto-detect per clip would be a different + #: workload. A dataset `language` column overrides this per row. + DEFAULT_LANGUAGE: ClassVar[str] = "en" + + _response_decoder: msgspec.json.Decoder = msgspec.json.Decoder( + TranscriptionResponse + ) + + @classmethod + def dataset_transforms(cls, model_params: ModelParams) -> list[Transform]: + metadata = { + "model": model_params.name, + "language": cls.DEFAULT_LANGUAGE, + "response_format": "json", + "temperature": model_params.temperature, + } + return [ + ColumnFilter( + required_columns=[AUDIO_FILE_COLUMN], + optional_columns=list(cls.FORM_FIELDS), + ), + # overwrite=False so a dataset row that carries its own `language` + # or `prompt` wins over the model_params defaults. + AddStaticColumns(metadata, overwrite=False), + ] + + @classmethod + def encode_query(cls, query: Query) -> bytes: + data = query.data + if AUDIO_FILE_COLUMN not in data: + raise KeyError( + f"{AUDIO_FILE_COLUMN!r} not found in query.data keys: " + f"{list(data.keys())}" + ) + audio_path = Path(data[AUDIO_FILE_COLUMN]) + fields = { + name: str(data[name]) + for name in cls.FORM_FIELDS + if data.get(name) is not None + } + return encode_multipart( + fields, audio_path.name, audio_path.read_bytes(), cls.MIME_TYPE + ) + + @classmethod + def decode_response(cls, response_bytes: bytes, query_id: str) -> QueryResult: + resp = cls._response_decoder.decode(response_bytes) + return QueryResult( + id=query_id, + response_output=TextModelOutput(output=resp.text), + ) + + @classmethod + def decode_sse_message(cls, json_bytes: bytes) -> str: + raise NotImplementedError( + "OpenAITranscriptionsAdapter does not use SSE streaming" + ) diff --git a/src/inference_endpoint/openai/types.py b/src/inference_endpoint/openai/types.py index 400baf704..fa791470e 100644 --- a/src/inference_endpoint/openai/types.py +++ b/src/inference_endpoint/openai/types.py @@ -270,3 +270,22 @@ class TextCompletionSSEMessage( """SSE message structure for /v1/completions streaming responses.""" choices: tuple[TextCompletionSSEChoice, ...] = () + + +# ============================================================================ +# OpenAI Audio Transcription Types (POST /v1/audio/transcriptions) +# Used by OpenAITranscriptionsAdapter for Whisper-class ASR endpoints. +# ============================================================================ + + +class TranscriptionResponse( + msgspec.Struct, frozen=True, kw_only=True, omit_defaults=True, gc=False +): # type: ignore[call-arg] + """Response body for `response_format=json`. + + The request is multipart/form-data, so there is no matching request struct - + the adapter encodes form fields directly. Server-specific extras (duration, + language, usage) decode away: msgspec ignores unknown fields. + """ + + text: str diff --git a/tests/integration/commands/test_probe_command.py b/tests/integration/commands/test_probe_command.py index a10885843..9321c37a9 100644 --- a/tests/integration/commands/test_probe_command.py +++ b/tests/integration/commands/test_probe_command.py @@ -23,6 +23,7 @@ - Error handling with failing endpoints """ +import cyclopts import pytest from inference_endpoint.commands.probe import ProbeConfig, execute_probe from inference_endpoint.config.schema import APIType @@ -98,6 +99,17 @@ def test_probe_rejects_videogen_api_type(self): with pytest.raises(InputValidationError, match="videogen"): execute_probe(config) + @pytest.mark.integration + def test_api_type_help_omits_unsupported_types(self): + """The --api-type help must not advertise an api_type the probe refuses.""" + param = next( + m + for m in ProbeConfig.model_fields["api_type"].metadata + if isinstance(m, cyclopts.Parameter) + ) + assert param.show_choices is False, "enum choices would list refused types" + assert APIType.OPENAI_TRANSCRIPTIONS.value not in (param.help or "") + @pytest.mark.integration def test_probe_with_invalid_endpoint(self): """Test probe fails gracefully with invalid endpoint.""" diff --git a/tests/integration/openai/test_transcriptions_adapter_integration.py b/tests/integration/openai/test_transcriptions_adapter_integration.py new file mode 100644 index 000000000..d73872b62 --- /dev/null +++ b/tests/integration/openai/test_transcriptions_adapter_integration.py @@ -0,0 +1,367 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""End-to-end transcription run: dataset -> worker -> multipart upload -> WER. + +Drives the real HTTP worker pool against a mock /v1/audio/transcriptions server, +so the multipart body, the adapter-declared Content-Type, and the WER scorer are +all exercised on the same path a real Whisper run takes. +""" + +import asyncio +import json +import random +import urllib.error +import urllib.request +from collections.abc import Generator +from pathlib import Path +from urllib.parse import urljoin + +import msgspec +import pytest +from aiohttp import web +from inference_endpoint import metrics +from inference_endpoint.config.runtime_settings import RuntimeSettings +from inference_endpoint.config.schema import ( + LoadPattern, + LoadPatternType, + ModelParams, +) +from inference_endpoint.core.record import EventRecord, EventType, SampleEventType +from inference_endpoint.core.types import ( + APIType, + Query, + QueryResult, + TextModelOutput, +) +from inference_endpoint.dataset_manager.dataset import Dataset +from inference_endpoint.dataset_manager.predefined.whisper_librispeech import ( + WhisperLibriSpeech, +) +from inference_endpoint.endpoint_client.config import HTTPClientConfig +from inference_endpoint.endpoint_client.http_client import HTTPEndpointClient +from inference_endpoint.endpoint_client.http_sample_issuer import HttpClientSampleIssuer +from inference_endpoint.evaluation.scoring import WERScorer +from inference_endpoint.load_generator.session import BenchmarkSession, PhaseConfig +from inference_endpoint.openai.transcriptions_adapter import ( + OpenAITranscriptionsAdapter, +) +from inference_endpoint.testing.echo_server import EchoServer +from transformers.models.whisper.english_normalizer import EnglishTextNormalizer + +MODEL = "openai/whisper-large-v3" +DATASET_NAME = "whisper_librispeech" + +# LibriSpeech transcripts are uppercase; the mock returns Whisper-style cased, +# punctuated text so the scorer's normalization is actually exercised. +CORPUS = [ + ( + "MISTER QUILTER IS THE APOSTLE OF THE MIDDLE CLASSES", + "Mister Quilter is the apostle of the middle classes.", + ), + ("AND WE ARE GLAD TO WELCOME HIS GOSPEL", "And we are glad to welcome his gospel."), + ( + "HE HAS GRAVE DOUBTS WHETHER SIR FREDERICK IS A GREAT PAINTER", + "He has grave doubts whether Sir Frederick is a great painter.", + ), + ("THE RAINBOW COMES AND GOES", "The rainbow comes and goes."), +] + + +#: 16-bit PCM WAV header plus a short run of samples. Only the bytes matter - +#: nothing in the client or the mock decodes the audio. +WAV_HEADER = ( + b"RIFF" + (36 + 512).to_bytes(4, "little") + b"WAVEfmt " + b"\x10\x00\x00\x00\x01\x00\x01\x00\x80>\x00\x00\x00}\x00\x00\x02\x00\x10\x00" + b"data" + (512).to_bytes(4, "little") +) + + +def write_wav(path: Path, payload: bytes | None = None) -> bytes: + """Write a dummy WAV at `path` and return its bytes.""" + data = WAV_HEADER + (payload if payload is not None else bytes(range(256)) * 2) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + return data + + +class MockTranscriptionServer(EchoServer): + """vLLM-shaped mock for `POST /v1/audio/transcriptions`. + + Subclasses `EchoServer` to reuse its background-thread aiohttp lifecycle + (port discovery, start/stop, ready event); only the route differs. The + upload is parsed with aiohttp's real multipart reader, so a malformed body + fails here rather than silently round-tripping. + + Returns the transcript registered for the uploaded file's basename, or the + basename itself when none is registered. Records each request's form fields + and uploaded bytes on `self.requests` so tests can assert on the wire + format rather than only on the decoded result. + """ + + def __init__(self, transcripts: dict[str, str] | None = None) -> None: + super().__init__(port=0) + self.transcripts = transcripts or {} + self.requests: list[dict] = [] + + def _register_routes(self, app: web.Application) -> None: + app.router.add_post("/v1/audio/transcriptions", self._handle_transcription) + + async def _handle_transcription(self, request: web.Request) -> web.Response: + if not request.content_type.startswith("multipart/form-data"): + return web.Response( + status=415, text=f"expected multipart, got {request.content_type}" + ) + + fields: dict[str, str] = {} + filename = "" + audio = b"" + reader = await request.multipart() + while True: + part = await reader.next() + if part is None: + break + if part.name == "file": + filename = part.filename or "" + audio = bytes(await part.read(decode=False)) + else: + fields[part.name] = (await part.read(decode=False)).decode() + + self.requests.append({"fields": fields, "filename": filename, "audio": audio}) + if not audio: + return web.Response(status=400, text="empty file part") + + text = self.transcripts.get(filename, filename) + if fields.get("response_format", "json") != "json": + return web.Response( + status=400, text="mock only serves response_format=json" + ) + return web.Response( + body=json.dumps({"text": text, "duration": 29.5}).encode(), + content_type="application/json", + ) + + +@pytest.fixture +def mock_transcription_server( + request: pytest.FixtureRequest, +) -> Generator[MockTranscriptionServer, None, None]: + transcripts = getattr(request, "param", None) + server = MockTranscriptionServer(transcripts=transcripts) + server.start() + yield server + server.stop() + + +class _NoOpPublisher: + def publish(self, event_record: EventRecord) -> None: + pass + + def flush(self) -> None: + pass + + +@pytest.fixture +def corpus(tmp_path: Path) -> tuple[Path, dict[str, bytes]]: + """Write an MLCommons-shaped manifest plus WAVs; return (manifest, bytes).""" + audio_dir = tmp_path / "dev-all-repack" + entries = [] + audio: dict[str, bytes] = {} + for i, (transcript, _) in enumerate(CORPUS): + name = f"{i}.wav" + # Distinct payload per clip so a mixed-up upload is detectable. + audio[name] = write_wav(audio_dir / name, bytes([i]) * 512) + entries.append( + { + "transcript": transcript, + "files": [ + {"fname": f"dev-all-repack/{name}", "speed": 1, "duration": 29.0} + ], + "original_duration": 29.0, + } + ) + manifest = tmp_path / "dev-all-repack.json" + manifest.write_text(json.dumps(entries)) + return manifest, audio + + +@pytest.fixture +def dataset(tmp_path: Path, corpus) -> Dataset: + manifest, _ = corpus + ds = WhisperLibriSpeech.get_dataloader( + datasets_dir=tmp_path / "cache", manifest=manifest + ) + ds.load( + api_type=APIType.OPENAI_TRANSCRIPTIONS, + model_params=ModelParams(name=MODEL, temperature=0.0), + ) + return ds + + +# Mock returns the Whisper-style transcript keyed by uploaded filename. +TRANSCRIPTS = {f"{i}.wav": hyp for i, (_, hyp) in enumerate(CORPUS)} + + +async def _run(url: str, dataset) -> tuple[dict[str, str], dict[str, int]]: + loop = asyncio.get_running_loop() + n_samples = dataset.num_samples() + responses: dict[str, str] = {} + + def on_complete(result: QueryResult) -> None: + responses[result.id] = result.get_response_output_string() + + http_config = HTTPClientConfig( + api_type=APIType.OPENAI_TRANSCRIPTIONS, + endpoint_urls=[ + urljoin( + url.rstrip("/") + "/", APIType.OPENAI_TRANSCRIPTIONS.default_route() + ) + ], + warmup_connections=0, + num_workers=2, + ) + client = await HTTPEndpointClient.create(http_config, loop) + try: + session = BenchmarkSession( + issuer=HttpClientSampleIssuer(client), + event_publisher=_NoOpPublisher(), + loop=loop, + on_sample_complete=on_complete, + ) + rt_settings = RuntimeSettings( + metrics.Throughput(100), + [metrics.Throughput(100)], + min_issue_duration_ms=0, + max_issue_duration_ms=60_000, + n_samples_from_dataset=n_samples, + n_samples_to_issue=n_samples, + min_sample_count=1, + rng_sched=random.Random(1234), + rng_sample_index=random.Random(1234), + load_pattern=LoadPattern(type=LoadPatternType.MAX_THROUGHPUT), + ) + result = await asyncio.wait_for( + session.run([PhaseConfig("performance", rt_settings, dataset)]), + timeout=120.0, + ) + finally: + await client.shutdown_async() + + assert result.perf_results[0].issued_count == n_samples + return responses, result.perf_results[0].uuid_to_index + + +@pytest.mark.integration +@pytest.mark.asyncio +@pytest.mark.parametrize("mock_transcription_server", [TRANSCRIPTS], indirect=True) +async def test_full_run_uploads_audio_and_scores_wer( + mock_transcription_server: MockTranscriptionServer, + dataset, + corpus, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + _, audio = corpus + responses, uuid_to_index = await _run(mock_transcription_server.url, dataset) + + # --- every sample completed and carried the right transcript back --- + assert len(responses) == len(CORPUS) + for sample_uuid, text in responses.items(): + _, expected_hypothesis = CORPUS[uuid_to_index[sample_uuid]] + assert text == expected_hypothesis + + # --- the server saw a well-formed upload per sample --- + assert len(mock_transcription_server.requests) == len(CORPUS) + seen = {} + for req in mock_transcription_server.requests: + assert req["fields"]["model"] == MODEL + assert req["fields"]["response_format"] == "json" + assert req["fields"]["temperature"] == "0.0" + assert "stream" not in req["fields"] + seen[req["filename"]] = req["audio"] + # Each clip's bytes arrived intact and with its own filename - a shared + # multipart boundary must not bleed one upload into another. + assert seen == audio + + # --- the real scorer over the real run artifacts --- + report_dir = tmp_path / "report" + report_dir.mkdir() + (report_dir / "sample_idx_map.json").write_bytes( + msgspec.json.encode({DATASET_NAME: uuid_to_index}) + ) + encoder = msgspec.json.Encoder(enc_hook=EventType.encode_hook) + with (report_dir / "events.jsonl").open("wb") as f: + for sample_uuid, text in responses.items(): + f.write( + encoder.encode( + EventRecord( + event_type=SampleEventType.COMPLETE, + sample_uuid=sample_uuid, + data=TextModelOutput(output=text), + ) + ) + + b"\n" + ) + + monkeypatch.setattr( + WERScorer, "_normalizer", lambda self: EnglishTextNormalizer({}) + ) + scorer = WERScorer( + DATASET_NAME, dataset, report_dir, ground_truth_column="transcript" + ) + score, n_repeats = scorer.score() + breakdown = scorer.score_breakdown() + assert breakdown is not None + + # Hypotheses differ from the references only in case and punctuation. + assert score == pytest.approx(100.0) + assert n_repeats == 1 + assert breakdown["edit_distance"] == 0 + assert breakdown["missing_samples"] == 0 + assert breakdown["reference_words"] == sum(len(r.split()) for r, _ in CORPUS) + + +@pytest.mark.integration +@pytest.mark.asyncio +async def test_server_rejects_a_non_multipart_body( + mock_transcription_server: MockTranscriptionServer, tmp_path: Path +) -> None: + """Guards the Content-Type wiring: JSON to this route is a 415.""" + url = f"{mock_transcription_server.url}/v1/audio/transcriptions" + req = urllib.request.Request( + url, + data=b'{"model": "m"}', + headers={"Content-Type": "application/json"}, + method="POST", + ) + with pytest.raises(urllib.error.HTTPError) as exc: + urllib.request.urlopen(req) + assert exc.value.code == 415 + + +@pytest.mark.integration +def test_dataset_rows_are_request_ready(dataset) -> None: + """The loaded row is used verbatim as Query.data, so it must encode.""" + row = dataset.load_sample(0) + assert set(row) == { + "audio_file", + "model", + "language", + "response_format", + "temperature", + } + assert row["language"] == "en" + body = OpenAITranscriptionsAdapter.encode_query(Query(id="q", data=row)) + assert Path(row["audio_file"]).read_bytes() in body diff --git a/tests/unit/async_utils/services/metrics_aggregator/test_token_metrics.py b/tests/unit/async_utils/services/metrics_aggregator/test_token_metrics.py index f42d0742c..b6a43ffa4 100644 --- a/tests/unit/async_utils/services/metrics_aggregator/test_token_metrics.py +++ b/tests/unit/async_utils/services/metrics_aggregator/test_token_metrics.py @@ -331,6 +331,37 @@ def apply_chat_template(self, *args, **kwargs): )[0] assert count > 0 + def test_baseline_failure_without_chat_template_is_not_an_error(self, caplog): + """ASR tokenizers (Whisper) have no chat template; baseline 0 is expected.""" + + class _NoTemplateTokenizer(_FakeTokenizer): + chat_template = None + + def apply_chat_template(self, *args, **kwargs): + raise ValueError("tokenizer.chat_template is not set") + + with patch(_MOCK_TARGET, _NoTemplateTokenizer): + with caplog.at_level(logging.DEBUG): + with BatchTokenizer("fake", n_workers=0, live_workers=1) as tok: + assert tok._baseline == 0 + records = [r for r in caplog.records if "baseline" in r.message] + assert records and all(r.levelno == logging.DEBUG for r in records) + + def test_baseline_failure_with_a_chat_template_is_logged_as_an_error(self, caplog): + """A tokenizer that has a template but fails still surfaces the problem.""" + + class _BrokenTemplateTokenizer(_FakeTokenizer): + chat_template = "{{ broken }}" + + def apply_chat_template(self, *args, **kwargs): + raise ValueError("render failed") + + with patch(_MOCK_TARGET, _BrokenTemplateTokenizer): + with caplog.at_level(logging.DEBUG): + with BatchTokenizer("fake", n_workers=0, live_workers=1) as tok: + assert tok._baseline == 0 + assert any(r.levelno == logging.ERROR for r in caplog.records) + def test_token_count_prompt_falls_back_on_template_error(self): class _TextOnlyTemplateTokenizer(_FakeTokenizerWithTemplate): def apply_chat_template(self, messages, **kwargs): diff --git a/tests/unit/config/test_schema.py b/tests/unit/config/test_schema.py index 9a881c184..230554553 100644 --- a/tests/unit/config/test_schema.py +++ b/tests/unit/config/test_schema.py @@ -1200,3 +1200,159 @@ def test_non_int_seed_rejected(self, register_temp_ruleset): ) with pytest.raises(ValidationError): self._submission(name) + + +class TestTranscriptionsAPIValidation: + """``openai_transcriptions`` constrains the run settings by api_type. + + /v1/audio/transcriptions has no SSE mode and no text prompt, so streaming, + ISL and warmup salt are not free choices here — a hand-written config gets + an error rather than a silently odd run. + """ + + def _cfg(self, **overrides) -> BenchmarkConfig: + kwargs: dict = { + "type": TestType.ONLINE, + "model_params": {"name": "openai/whisper-large-v3"}, + "endpoint_config": { + "endpoints": ["http://localhost:8000"], + "api_type": APIType.OPENAI_TRANSCRIPTIONS, + }, + "datasets": [{"path": "clips.jsonl"}], + "settings": {"load_pattern": {"type": "poisson", "target_qps": 1}}, + } + kwargs.update(overrides) + return BenchmarkConfig(**kwargs) + + @pytest.mark.unit + def test_auto_streaming_resolves_off_even_online(self): + # The generic rule would resolve AUTO to ON for an online run. + assert self._cfg().model_params.streaming == StreamingMode.OFF + + @pytest.mark.unit + def test_explicit_streaming_on_is_rejected(self): + with pytest.raises(ValueError, match="no streaming mode"): + self._cfg( + model_params={"name": "whisper", "streaming": "on"}, + ) + + @pytest.mark.unit + def test_explicit_streaming_off_is_accepted(self): + cfg = self._cfg(model_params={"name": "whisper", "streaming": "off"}) + assert cfg.model_params.streaming == StreamingMode.OFF + + @pytest.mark.unit + def test_metrics_isl_is_forced_off(self): + # Default is True; an audio upload has no prompt to tokenize, so the + # series could only ever stay empty. + assert self._cfg().settings.metrics_isl is False + + @pytest.mark.unit + def test_metrics_isl_stays_off_when_set_explicitly(self): + cfg = self._cfg( + settings={ + "load_pattern": {"type": "poisson", "target_qps": 1}, + "metrics_isl": False, + } + ) + assert cfg.settings.metrics_isl is False + + @pytest.mark.unit + def test_warmup_salt_is_rejected(self): + with pytest.raises(ValueError, match="warmup.salt"): + self._cfg( + settings={ + "load_pattern": {"type": "poisson", "target_qps": 1}, + "warmup": {"enabled": True, "salt": True}, + } + ) + + @pytest.mark.unit + def test_warmup_without_salt_is_accepted(self): + cfg = self._cfg( + settings={ + "load_pattern": {"type": "poisson", "target_qps": 1}, + "warmup": {"enabled": True, "salt": False}, + } + ) + assert cfg.settings.warmup.enabled is True + + @pytest.mark.unit + def test_other_api_types_keep_online_streaming(self): + # The constraint is keyed on api_type, not applied to every run. + cfg = BenchmarkConfig( + type=TestType.ONLINE, + model_params={"name": "M"}, + endpoint_config={ + "endpoints": ["http://localhost:8000"], + "api_type": APIType.OPENAI, + }, + datasets=[{"path": "d.jsonl"}], + settings={"load_pattern": {"type": "poisson", "target_qps": 1}}, + ) + assert cfg.model_params.streaming == StreamingMode.ON + assert cfg.settings.metrics_isl is True + + +class TestNonStreamingAPITypeCapabilities: + """The two capabilities are independent, and videogen proves it. + + ``supports_streaming`` and ``has_text_prompt`` gate different constraints. + videogen is non-streaming *and* text-to-video, so it takes the streaming + rules and none of the prompt ones. + """ + + def _videogen(self, **overrides) -> BenchmarkConfig: + kwargs: dict = { + "type": TestType.ONLINE, + "model_params": {"name": "wan2.2-t2v-a14b"}, + "endpoint_config": { + "endpoints": ["http://localhost:8000"], + "api_type": APIType.VIDEOGEN, + }, + "datasets": [{"path": "prompts.jsonl"}], + "settings": {"load_pattern": {"type": "poisson", "target_qps": 1}}, + } + kwargs.update(overrides) + return BenchmarkConfig(**kwargs) + + @pytest.mark.unit + @pytest.mark.parametrize( + ("api_type", "streams", "prompts"), + [ + (APIType.OPENAI, True, True), + (APIType.OPENAI_COMPLETIONS, True, True), + (APIType.SGLANG, True, True), + (APIType.VIDEOGEN, False, True), + (APIType.OPENAI_TRANSCRIPTIONS, False, False), + ], + ) + def test_capability_flags(self, api_type, streams, prompts): + assert api_type.supports_streaming is streams + assert api_type.has_text_prompt is prompts + + @pytest.mark.unit + def test_videogen_auto_streaming_resolves_off_even_online(self): + assert self._videogen().model_params.streaming == StreamingMode.OFF + + @pytest.mark.unit + def test_videogen_streaming_on_is_rejected(self): + with pytest.raises(ValueError, match="no streaming mode"): + self._videogen( + model_params={"name": "wan", "streaming": "on"}, + ) + + @pytest.mark.unit + def test_videogen_keeps_metrics_isl(self): + # A text-to-video prompt is a real text prompt: ISL is measurable. + assert self._videogen().settings.metrics_isl is True + + @pytest.mark.unit + def test_videogen_keeps_warmup_salt(self): + cfg = self._videogen( + settings={ + "load_pattern": {"type": "poisson", "target_qps": 1}, + "warmup": {"enabled": True, "salt": True}, + } + ) + assert cfg.settings.warmup.salt is True diff --git a/tests/unit/dataset_manager/test_whisper_librispeech_dataset.py b/tests/unit/dataset_manager/test_whisper_librispeech_dataset.py new file mode 100644 index 000000000..3939fcae6 --- /dev/null +++ b/tests/unit/dataset_manager/test_whisper_librispeech_dataset.py @@ -0,0 +1,380 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for the WhisperLibriSpeech predefined dataset. + +The download tests run without network: requests.get (downloader script fetch) +and subprocess.run (the mlc-r2-downloader invocation) are mocked in the loader's +own module, so only its logic runs. +""" + +import json +import subprocess +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest +from inference_endpoint.dataset_manager.dataset import Dataset +from inference_endpoint.dataset_manager.predefined.whisper_librispeech import ( + MANIFEST_FILENAME, + SOURCE_ENV, + WhisperLibriSpeech, +) + +_MODULE = "inference_endpoint.dataset_manager.predefined.whisper_librispeech" + + +def _entry(fname: str, transcript: str, duration: float = 29.5) -> dict: + """One MLCommons dev-all-repack.json entry.""" + return { + "transcript": transcript, + "files": [{"fname": fname, "speed": 1, "duration": duration}], + "original_duration": duration, + "original_num_samples": int(duration * 16000), + } + + +@pytest.fixture +def manifest(tmp_path: Path) -> Path: + path = tmp_path / "dev-all-repack.json" + path.write_text( + json.dumps( + [ + _entry("dev-all-repack/0.wav", "mister quilter is the apostle", 29.1), + _entry("dev-all-repack/1.wav", "of the middle classes", 28.4), + ] + ) + ) + return path + + +def _mock_requests_get() -> MagicMock: + """A requests.get returning a trivial downloader script body.""" + resp = MagicMock() + resp.content = b"#!/usr/bin/env bash\nexit 0\n" + resp.raise_for_status.return_value = None + return resp + + +def _fake_run_unpacking(cmd, **kwargs) -> MagicMock: + """subprocess.run side effect that unpacks a corpus under the -d directory.""" + download_dir = Path(cmd[cmd.index("-d") + 1]) + (download_dir / "dev-all-repack").mkdir(parents=True, exist_ok=True) + (download_dir / "dev-all-repack" / "0.wav").write_bytes(b"RIFF") + (download_dir / MANIFEST_FILENAME).write_text( + json.dumps([_entry("dev-all-repack/0.wav", "downloaded transcript")]) + ) + return MagicMock(returncode=0, stderr="") + + +@pytest.fixture +def no_local_manifest(monkeypatch: pytest.MonkeyPatch) -> None: + """Drop the env overrides so generate() takes the download path.""" + monkeypatch.delenv(WhisperLibriSpeech.R2_DATASET_URI_ENV, raising=False) + monkeypatch.delenv(SOURCE_ENV, raising=False) + + +@pytest.mark.unit +class TestRegistration: + def test_registered_under_its_dataset_id(self): + assert Dataset.PREDEFINED["whisper_librispeech"] is WhisperLibriSpeech + + def test_serves_both_phases(self): + # ACCURACY_ONLY would make the perf phase reject this dataset. + assert WhisperLibriSpeech.ACCURACY_ONLY is False + + def test_column_names(self): + assert WhisperLibriSpeech.COLUMN_NAMES == [ + "audio_file", + "transcript", + "duration", + ] + + +@pytest.mark.unit +class TestGenerate: + def test_flattens_manifest_entries(self, tmp_path: Path, manifest: Path): + df = WhisperLibriSpeech.generate(tmp_path / "cache", manifest=manifest) + assert list(df.columns) == WhisperLibriSpeech.COLUMN_NAMES + assert len(df) == 2 + assert df.iloc[0]["transcript"] == "mister quilter is the apostle" + assert df.iloc[0]["duration"] == pytest.approx(29.1) + + def test_resolves_fname_against_the_manifest_directory( + self, tmp_path: Path, manifest: Path + ): + df = WhisperLibriSpeech.generate(tmp_path / "cache", manifest=manifest) + assert df.iloc[0]["audio_file"] == str(manifest.parent / "dev-all-repack/0.wav") + + def test_explicit_data_dir_overrides_the_manifest_directory( + self, tmp_path: Path, manifest: Path + ): + df = WhisperLibriSpeech.generate( + tmp_path / "cache", manifest=manifest, data_dir="/mnt/audio" + ) + assert df.iloc[0]["audio_file"] == "/mnt/audio/dev-all-repack/0.wav" + + def test_drops_fname_prefix_absent_from_the_corpus_root(self, tmp_path: Path): + # The hosted artifact unpacks dev-all-repack/ into the corpus root, + # while the manifest's fnames keep the upstream pipeline's ./data/ + # prefix. The clips that exist on disk are what decide the layout. + source = tmp_path / "source" + (source / "dev-all-repack").mkdir(parents=True) + for name in ("0.wav", "1.wav"): + (source / "dev-all-repack" / name).write_bytes(b"RIFF") + path = source / MANIFEST_FILENAME + path.write_text( + json.dumps( + [ + _entry("./data/dev-all-repack/0.wav", "first"), + _entry("./data/dev-all-repack/1.wav", "second"), + ] + ) + ) + + df = WhisperLibriSpeech.generate(tmp_path / "cache", manifest=path) + assert df.iloc[0]["audio_file"] == str(source / "dev-all-repack/0.wav") + assert df.iloc[1]["audio_file"] == str(source / "dev-all-repack/1.wav") + + def test_keeps_fname_as_written_when_no_clip_resolves(self, tmp_path: Path): + path = tmp_path / MANIFEST_FILENAME + path.write_text(json.dumps([_entry("./data/dev-all-repack/0.wav", "first")])) + + df = WhisperLibriSpeech.generate(tmp_path / "cache", manifest=path) + assert df.iloc[0]["audio_file"] == str(tmp_path / "data/dev-all-repack/0.wav") + + def test_writes_and_reuses_the_parquet_cache(self, tmp_path: Path, manifest: Path): + cache = tmp_path / "cache" + WhisperLibriSpeech.generate(cache, manifest=manifest) + parquet = cache / "whisper_librispeech" / "whisper_librispeech.parquet" + assert parquet.exists() + + # A second call must not need the manifest at all. + manifest.unlink() + df = WhisperLibriSpeech.generate(cache, manifest=manifest) + assert len(df) == 2 + + def test_force_rebuilds_from_the_manifest(self, tmp_path: Path, manifest: Path): + cache = tmp_path / "cache" + WhisperLibriSpeech.generate(cache, manifest=manifest) + manifest.write_text(json.dumps([_entry("dev-all-repack/0.wav", "only one")])) + df = WhisperLibriSpeech.generate(cache, manifest=manifest, force=True) + assert len(df) == 1 + assert df.iloc[0]["transcript"] == "only one" + + def test_max_samples_truncates_in_manifest_order( + self, tmp_path: Path, manifest: Path + ): + df = WhisperLibriSpeech.generate( + tmp_path / "cache", manifest=manifest, max_samples=1 + ) + assert len(df) == 1 + assert df.iloc[0]["transcript"] == "mister quilter is the apostle" + + def test_reads_the_manifest_from_the_environment( + self, tmp_path: Path, manifest: Path, monkeypatch: pytest.MonkeyPatch + ): + monkeypatch.setenv(SOURCE_ENV, str(manifest)) + df = WhisperLibriSpeech.generate(tmp_path / "cache") + assert len(df) == 2 + + +@pytest.mark.unit +class TestR2Download: + def test_downloads_when_no_manifest_is_supplied( + self, tmp_path: Path, no_local_manifest: None + ): + with ( + patch(f"{_MODULE}.requests.get", return_value=_mock_requests_get()), + patch(f"{_MODULE}.subprocess.run", side_effect=_fake_run_unpacking), + ): + df = WhisperLibriSpeech.generate(tmp_path / "cache") + + assert df.iloc[0]["transcript"] == "downloaded transcript" + # fname resolves against the downloaded manifest's own directory. + assert df.iloc[0]["audio_file"] == str( + tmp_path / "cache/whisper_librispeech/source/dev-all-repack/0.wav" + ) + + def test_a_local_manifest_skips_the_download(self, tmp_path: Path, manifest: Path): + with patch(f"{_MODULE}.requests.get") as get: + WhisperLibriSpeech.generate(tmp_path / "cache", manifest=manifest) + get.assert_not_called() + + def test_force_reuses_the_download_instead_of_refetching( + self, tmp_path: Path, no_local_manifest: None + ): + cache = tmp_path / "cache" + with ( + patch(f"{_MODULE}.requests.get", return_value=_mock_requests_get()), + patch(f"{_MODULE}.subprocess.run", side_effect=_fake_run_unpacking), + ): + WhisperLibriSpeech.generate(cache) + + # force rebuilds the parquet; it must not re-fetch GBs of audio. + with patch(f"{_MODULE}.requests.get") as get: + assert len(WhisperLibriSpeech.generate(cache, force=True)) == 1 + get.assert_not_called() + + @pytest.mark.parametrize( + "uri", + [ + "https://evil.example.com/x.uri", + "http://inference.mlcommons-storage.org/x.uri", + ], + ) + def test_untrusted_uri_is_refused_before_any_fetch( + self, + tmp_path: Path, + uri: str, + no_local_manifest: None, + monkeypatch: pytest.MonkeyPatch, + ): + monkeypatch.setenv(WhisperLibriSpeech.R2_DATASET_URI_ENV, uri) + with patch(f"{_MODULE}.requests.get") as get: + with pytest.raises(ValueError, match="untrusted URI"): + WhisperLibriSpeech.generate(tmp_path / "cache") + get.assert_not_called() + + def test_downloader_failure_is_reported( + self, tmp_path: Path, no_local_manifest: None + ): + with ( + patch(f"{_MODULE}.requests.get", return_value=_mock_requests_get()), + patch( + f"{_MODULE}.subprocess.run", + return_value=MagicMock(returncode=2, stderr="r2 boom"), + ), + ): + with pytest.raises(RuntimeError, match="r2 boom"): + WhisperLibriSpeech.generate(tmp_path / "cache") + + def test_downloader_is_bounded_by_a_timeout( + self, tmp_path: Path, no_local_manifest: None + ): + # Dataset generation runs before RunWatchdog arms the whole-run + # deadline, so a stalled multi-GB fetch has nothing else to stop it. + with ( + patch(f"{_MODULE}.requests.get", return_value=_mock_requests_get()), + patch(f"{_MODULE}.subprocess.run") as run, + ): + run.side_effect = _fake_run_unpacking + WhisperLibriSpeech.generate(tmp_path / "cache") + assert ( + run.call_args.kwargs["timeout"] == WhisperLibriSpeech.R2_DOWNLOAD_TIMEOUT_S + ) + + def test_a_stalled_download_raises_rather_than_hanging( + self, tmp_path: Path, no_local_manifest: None + ): + with ( + patch(f"{_MODULE}.requests.get", return_value=_mock_requests_get()), + patch( + f"{_MODULE}.subprocess.run", + side_effect=subprocess.TimeoutExpired(cmd="bash", timeout=1), + ), + ): + with pytest.raises(RuntimeError, match="exceeded"): + WhisperLibriSpeech.generate(tmp_path / "cache") + + def test_the_downloader_script_is_removed_after_a_timeout( + self, tmp_path: Path, no_local_manifest: None + ): + cache = tmp_path / "cache" + with ( + patch(f"{_MODULE}.requests.get", return_value=_mock_requests_get()), + patch( + f"{_MODULE}.subprocess.run", + side_effect=subprocess.TimeoutExpired(cmd="bash", timeout=1), + ), + ): + with pytest.raises(RuntimeError): + WhisperLibriSpeech.generate(cache) + assert not list(cache.rglob("mlc-r2-downloader.sh")) + + +@pytest.mark.unit +class TestGenerateErrors: + def test_absent_manifest_path(self, tmp_path: Path): + with pytest.raises(FileNotFoundError, match="not found at"): + WhisperLibriSpeech.generate( + tmp_path / "cache", manifest=tmp_path / "missing.json" + ) + + def test_non_json_source_is_rejected(self, tmp_path: Path): + raw = tmp_path / "dev-all.flac" + raw.write_bytes(b"\x00") + with pytest.raises(NotImplementedError, match="dev-all-repack.json"): + WhisperLibriSpeech.generate(tmp_path / "cache", manifest=raw) + + def test_entry_without_files_is_rejected(self, tmp_path: Path): + path = tmp_path / "bad.json" + path.write_text(json.dumps([{"transcript": "hi", "original_duration": 1.0}])) + with pytest.raises(ValueError, match="no 'files' list"): + WhisperLibriSpeech.generate(tmp_path / "cache", manifest=path) + + def test_speed_perturbed_only_entry_is_rejected(self, tmp_path: Path): + path = tmp_path / "perturbed.json" + path.write_text( + json.dumps( + [ + { + "transcript": "hi", + "files": [ + {"fname": "slow.wav", "speed": 0.9, "duration": 1.0}, + {"fname": "fast.wav", "speed": 1.1, "duration": 1.0}, + ], + "original_duration": 1.0, + } + ] + ) + ) + with pytest.raises(ValueError, match="speed == 1"): + WhisperLibriSpeech.generate(tmp_path / "cache", manifest=path) + + def test_unperturbed_clip_is_selected(self, tmp_path: Path): + path = tmp_path / "mixed.json" + path.write_text( + json.dumps( + [ + { + "transcript": "hi", + "files": [ + {"fname": "slow.wav", "speed": 0.9, "duration": 1.1}, + {"fname": "normal.wav", "speed": 1, "duration": 1.0}, + ], + "original_duration": 1.0, + } + ] + ) + ) + df = WhisperLibriSpeech.generate(tmp_path / "cache", manifest=path) + assert df.iloc[0]["audio_file"].endswith("normal.wav") + + def test_sidecar_transcript_manifest_is_rejected(self, tmp_path: Path): + path = tmp_path / "sidecar.json" + path.write_text( + json.dumps( + [ + { + "text_filepath": "0.txt", + "files": [{"fname": "0.wav", "speed": 1, "duration": 1.0}], + "original_duration": 1.0, + } + ] + ) + ) + with pytest.raises(ValueError, match="text_filepath"): + WhisperLibriSpeech.generate(tmp_path / "cache", manifest=path) diff --git a/tests/unit/endpoint_client/test_http.py b/tests/unit/endpoint_client/test_http.py index 40e2dc5e6..363af9c43 100644 --- a/tests/unit/endpoint_client/test_http.py +++ b/tests/unit/endpoint_client/test_http.py @@ -171,6 +171,74 @@ def test_cache_headers_pre_caches(self): assert b"Content-Length: 17" in request +class TestHttpRequestTemplateContentType: + """Adapter-declared Content-Type (e.g. a multipart audio upload).""" + + MULTIPART = "multipart/form-data; boundary=----boundary-With;Punctuation=1" + + @staticmethod + def _parse(request: bytes) -> tuple[dict[str, list[str]], bytes]: + state: dict[str, list[str]] = {} + body = bytearray() + + class Handler: + def on_url(self, url): + pass + + def on_header(self, name, value): + state.setdefault(name.decode().lower(), []).append(value.decode()) + + def on_body(self, chunk): + body.extend(chunk) + + httptools.HttpRequestParser(Handler()).feed_data(request) + return state, bytes(body) + + def test_default_is_json(self): + template = HttpRequestTemplate.from_url("localhost", 8080, "/v1/chat") + assert template.content_type == "application/json" + + @pytest.mark.parametrize("streaming", [False, True]) + def test_declared_content_type_renders_exactly_once(self, streaming): + template = HttpRequestTemplate.from_url( + "localhost", 8080, "/v1/audio/transcriptions", content_type=self.MULTIPART + ) + headers, _ = self._parse(template.build_request(b"--x--\r\n", streaming)) + # A duplicated Content-Type (the failure mode of appending via + # extra_headers) makes the server read the wrong media type. + assert headers["content-type"] == [self.MULTIPART] + + def test_boundary_punctuation_does_not_break_header_framing(self): + """`;` and `=` inside the Content-Type value must stay in one header.""" + template = HttpRequestTemplate.from_url( + "localhost", 8080, "/v1/audio/transcriptions", content_type=self.MULTIPART + ) + body = b'--x\r\nContent-Disposition: form-data; name="a"\r\n\r\n1\r\n--x--\r\n' + headers, parsed_body = self._parse(template.build_request(body, False)) + assert headers["content-type"] == [self.MULTIPART] + assert headers["content-length"] == [str(len(body))] + assert parsed_body == body + + def test_binary_body_is_sent_verbatim(self): + """A non-JSON body passes through untouched with a correct length.""" + template = HttpRequestTemplate.from_url( + "localhost", 8080, "/v1/audio/transcriptions", content_type=self.MULTIPART + ) + body = bytes(range(256)) * 4 + headers, parsed_body = self._parse(template.build_request(body, False)) + assert parsed_body == body + assert headers["content-length"] == [str(len(body))] + + def test_cached_headers_preserve_the_content_type(self): + template = HttpRequestTemplate.from_url( + "localhost", 8080, "/v1/audio/transcriptions", content_type=self.MULTIPART + ) + template.cache_headers({"Authorization": "Bearer t"}) + headers, _ = self._parse(template.build_request(b"x", False)) + assert headers["content-type"] == [self.MULTIPART] + assert headers["authorization"] == ["Bearer t"] + + # ============================================================================= # HttpResponseProtocol Tests # ============================================================================= diff --git a/tests/unit/evaluation/test_wer.py b/tests/unit/evaluation/test_wer.py new file mode 100644 index 000000000..bcdde70c5 --- /dev/null +++ b/tests/unit/evaluation/test_wer.py @@ -0,0 +1,201 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for the MLPerf speech2text WER metric.""" + +import pytest +from inference_endpoint.evaluation.wer import ( + LABELS, + levenshtein, + project_to_labels, + wer_counts, +) + + +@pytest.mark.unit +class TestLevenshtein: + @pytest.mark.parametrize( + ("a", "b", "expected"), + [ + ([], [], 0), + (["a"], [], 1), + ([], ["a"], 1), + (["a", "b", "c"], ["a", "b", "c"], 0), + (["a", "b", "c"], ["a", "b", "d"], 1), + (list("kitten"), list("sitting"), 3), + (["a", "b"], ["b", "a"], 2), + ], + ) + def test_distance(self, a, b, expected): + assert levenshtein(a, b) == expected + + def test_is_symmetric(self): + a, b = list("saturday"), list("sunday") + assert levenshtein(a, b) == levenshtein(b, a) + + +@pytest.mark.unit +class TestProjectToLabels: + def test_lowercases(self): + assert project_to_labels("MISTER QUILTER") == "mister quilter" + + def test_drops_punctuation_outside_the_inventory(self): + assert project_to_labels("Hello, world! (yes)") == "hello world yes" + + def test_keeps_apostrophes_digits_and_symbols(self): + assert project_to_labels("IT'S 42% +3 -1 $5") == "it's 42% +3 -1 $5" + + def test_inventory_matches_the_reference_label_set(self): + assert LABELS == frozenset(" abcdefghijklmnopqrstuvwxyz'0123456789$%+-") + + +@pytest.mark.unit +class TestWerCounts: + def test_exact_match_scores_zero_errors(self): + words = ["the", "cat", "sat"] + assert wer_counts(words, words) == (0, 3) + + def test_substitution_counts_one_error(self): + assert wer_counts(["the", "dog", "sat"], ["the", "cat", "sat"]) == (1, 3) + + def test_deletion_counts_one_error(self): + assert wer_counts(["the", "sat"], ["the", "cat", "sat"]) == (1, 3) + + def test_insertion_counts_one_error(self): + assert wer_counts(["the", "big", "cat", "sat"], ["the", "cat", "sat"]) == (1, 3) + + def test_empty_hypothesis_counts_every_reference_word(self): + assert wer_counts([], ["the", "cat", "sat"]) == (3, 3) + + def test_empty_reference_reports_zero_words(self): + # Denominator of 0 - the caller must sum counts across the corpus before + # dividing, which is why this returns counts rather than a rate. + assert wer_counts(["the"], []) == (1, 0) + + def test_both_empty(self): + assert wer_counts([], []) == (0, 0) + + def test_joined_hypothesis_word_is_credited(self): + # "can not" -> "cannot" is the concatenation case a generic WER library + # would score as 2 errors. + assert wer_counts(["cannot"], ["can", "not"]) == (0, 2) + + def test_joined_reference_word_is_credited(self): + assert wer_counts(["can", "not"], ["cannot"]) == (0, 1) + + def test_three_word_join_is_credited(self): + assert wer_counts(["anyoneelse"], ["any", "one", "else"]) == (0, 3) + + def test_four_word_join_exceeds_the_span_and_counts_errors(self): + # Pinned against upstream: 4 errors over 4 words, not a partial credit. + assert wer_counts(["abcd"], ["a", "b", "c", "d"]) == (4, 4) + + def test_join_credited_mid_sentence(self): + assert wer_counts(["i", "cannot", "go"], ["i", "can", "not", "go"]) == (0, 4) + + def test_corpus_wer_is_summed_not_averaged(self): + samples = [ + (["the", "dog"], ["the", "cat"]), + (["a"] * 10, ["a"] * 10), + ] + distance = words = 0 + for hyp, ref in samples: + d, w = wer_counts(hyp, ref) + distance += d + words += w + # 1 error over 12 reference words, not the mean of 50% and 0%. + assert (distance, words) == (1, 12) + + +@pytest.mark.unit +class TestEndToEndOnText: + def test_reference_pipeline_on_a_librispeech_style_pair(self): + reference = "MISTER QUILTER IS THE APOSTLE OF THE MIDDLE CLASSES" + hypothesis = "Mister Quilter is the apostle of the middle classes." + hyp = project_to_labels(hypothesis).split() + ref = project_to_labels(reference).split() + assert wer_counts(hyp, ref) == (0, 9) + + +#: ``(hypothesis, reference, (edit_distance, reference_words))`` captured by +#: running the MLCommons reference implementation directly — its +#: ``speech2text/helpers.py:compute_wer_with_concatenation`` composed with +#: ``legacy_helpers.py:__levenshtein``, pinned at mlcommons/inference commit +#: 3fbc329939999c13d0a7b5e67fb2092287e06047. These are the upstream numbers, not +#: this implementation's output, so they fail if the port drifts. +#: +#: Generated alongside a randomized differential sweep (20k trials over a small +#: alphabet chosen so concatenations actually occur) that found zero divergence. +#: The cases below are the ones where the metric's behaviour is load-bearing: +#: joins in each direction, a join past the 3-word span, and the empty edges. +UPSTREAM_VECTORS = [ + (["the", "cat", "sat"], ["the", "cat", "sat"], (0, 3)), + (["the", "dog", "sat"], ["the", "cat", "sat"], (1, 3)), + (["the", "sat"], ["the", "cat", "sat"], (1, 3)), + (["the", "big", "cat", "sat"], ["the", "cat", "sat"], (1, 3)), + ([], ["the", "cat", "sat"], (3, 3)), + (["the"], [], (1, 0)), + ([], [], (0, 0)), + (["cannot"], ["can", "not"], (0, 2)), + (["can", "not"], ["cannot"], (0, 1)), + (["anyoneelse"], ["any", "one", "else"], (0, 3)), + (["any", "one", "else"], ["anyoneelse"], (0, 1)), + (["abcd"], ["a", "b", "c", "d"], (4, 4)), + (["i", "cannot", "go"], ["i", "can", "not", "go"], (0, 4)), + ( + ["mister", "quilter", "is", "the", "apostle"], + ["mister", "quilter", "is", "the", "apostle"], + (0, 5), + ), + ( + ["mister", "quilter", "is", "the", "epistle"], + ["mister", "quilter", "is", "the", "apostle"], + (1, 5), + ), + ( + ["nor", "is", "mister", "quilters", "manner", "less", "interesting"], + ["nor", "is", "mister", "quilter", "s", "manner", "less", "interesting"], + (0, 8), + ), + ( + ["goodmorning", "everyone", "welcome"], + ["good", "morning", "every", "one", "welcome"], + (0, 5), + ), + (["a", "bc", "d"], ["a", "b", "c", "d"], (0, 4)), + (["abc", "d"], ["a", "b", "c", "d"], (0, 4)), + (["x", "abc"], ["x", "a", "b", "c"], (0, 4)), + ( + ["and", "we", "are", "glad", "to", "welcome", "his", "gospel"], + ["and", "we", "are", "glad", "to", "welcome", "his", "gospel"], + (0, 8), + ), +] + + +@pytest.mark.unit +class TestUpstreamParity: + @pytest.mark.parametrize(("hypothesis", "reference", "expected"), UPSTREAM_VECTORS) + def test_matches_the_reference_implementation( + self, hypothesis, reference, expected + ): + assert wer_counts(hypothesis, reference) == tuple(expected) + + def test_corpus_wer_over_the_whole_vector_set(self): + # The aggregate the benchmark actually reports: summed, then divided. + distance = sum(e[0] for _, _, e in UPSTREAM_VECTORS) + words = sum(e[1] for _, _, e in UPSTREAM_VECTORS) + ours = [wer_counts(h, r) for h, r, _ in UPSTREAM_VECTORS] + assert (sum(d for d, _ in ours), sum(w for _, w in ours)) == (distance, words) diff --git a/tests/unit/evaluation/test_wer_scorer.py b/tests/unit/evaluation/test_wer_scorer.py new file mode 100644 index 000000000..1e400dae1 --- /dev/null +++ b/tests/unit/evaluation/test_wer_scorer.py @@ -0,0 +1,342 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for WERScorer.""" + +from pathlib import Path +from unittest.mock import MagicMock + +import msgspec +import pandas as pd +import pytest +from inference_endpoint.core.record import EventRecord, EventType, SampleEventType +from inference_endpoint.core.types import TextModelOutput +from inference_endpoint.evaluation import scoring +from inference_endpoint.evaluation.extractor import PythonCodeExtractor +from inference_endpoint.evaluation.scoring import Scorer, WERScorer +from inference_endpoint.evaluation.wer import project_to_labels +from inference_endpoint.exceptions import InputValidationError +from transformers.models.whisper.english_normalizer import EnglishTextNormalizer + +DATASET_NAME = "whisper_librispeech" + +REFERENCES = [ + "mister quilter is the apostle of the middle classes", + "and we are glad to welcome his gospel", + "nor is mister quilter's manner less interesting than his matter", +] + + +def _ref_word_count(reference: str) -> int: + """Reference word count as the scorer counts it (post-normalization).""" + normalizer = EnglishTextNormalizer({}) + return len(normalizer(project_to_labels(reference)).split()) + + +@pytest.fixture +def hermetic_normalizer(monkeypatch: pytest.MonkeyPatch): + """Use the real normalizer with an empty spelling map - no HF download.""" + monkeypatch.setattr( + WERScorer, "_normalizer", lambda self: EnglishTextNormalizer({}) + ) + + +@pytest.fixture +def dataset(): + ds = MagicMock() + ds.dataframe = pd.DataFrame({"transcript": REFERENCES}) + ds.num_samples.return_value = len(REFERENCES) + return ds + + +def _stage(report_dir: Path, outputs: dict[int, str]) -> Path: + """Write sample_idx_map for all references and events for `outputs` only. + + Keys are dataset sample indices; a reference without an entry stands in for + a sample that never produced a COMPLETE event. + """ + report_dir.mkdir(exist_ok=True) + uuids = {i: f"uuid-{i}" for i in range(len(REFERENCES))} + (report_dir / "sample_idx_map.json").write_bytes( + msgspec.json.encode({DATASET_NAME: {u: i for i, u in uuids.items()}}) + ) + encoder = msgspec.json.Encoder(enc_hook=EventType.encode_hook) + with (report_dir / "events.jsonl").open("wb") as f: + for index, text in outputs.items(): + rec = EventRecord( + event_type=SampleEventType.COMPLETE, + sample_uuid=uuids[index], + data=TextModelOutput(output=text), + ) + f.write(encoder.encode(rec) + b"\n") + return report_dir + + +def _scorer(report_dir: Path, dataset, **kwargs) -> WERScorer: + return WERScorer( + DATASET_NAME, + dataset, + report_dir, + ground_truth_column="transcript", + **kwargs, + ) + + +@pytest.mark.unit +class TestRegistration: + def test_registered(self): + assert "wer" in Scorer.available_scorers() + assert Scorer.get("wer") is WERScorer + + def test_requires_no_extractor(self): + assert WERScorer.REQUIRES_EXTRACTOR is False + + def test_runs_on_endpoint_outputs(self): + assert WERScorer.SKIP_ENDPOINT_PHASE is False + + +@pytest.mark.unit +@pytest.mark.usefixtures("hermetic_normalizer") +class TestScore: + def test_perfect_transcription_scores_100(self, tmp_path: Path, dataset): + report_dir = _stage(tmp_path / "r", dict(enumerate(REFERENCES))) + score, n_repeats = _scorer(report_dir, dataset).score() + assert score == pytest.approx(100.0) + assert n_repeats == 1 + + def test_casing_and_punctuation_do_not_count_as_errors( + self, tmp_path: Path, dataset + ): + outputs = {i: f"{r.capitalize()}." for i, r in enumerate(REFERENCES)} + report_dir = _stage(tmp_path / "r", outputs) + assert _scorer(report_dir, dataset).score()[0] == pytest.approx(100.0) + + def test_one_substitution_over_the_corpus(self, tmp_path: Path, dataset): + outputs = dict(enumerate(REFERENCES)) + outputs[0] = REFERENCES[0].replace("apostle", "epistle") + report_dir = _stage(tmp_path / "r", outputs) + + scorer = _scorer(report_dir, dataset) + score, _ = scorer.score() + breakdown = scorer.score_breakdown() + assert breakdown is not None + + assert breakdown["edit_distance"] == 1 + # Corpus-level: 1 error over every reference word, not a per-sample mean. + assert breakdown["wer_percent"] == pytest.approx( + 100.0 / breakdown["reference_words"] + ) + assert score == pytest.approx(100.0 - breakdown["wer_percent"]) + + def test_empty_transcription_counts_every_word_as_an_error( + self, tmp_path: Path, dataset + ): + report_dir = _stage(tmp_path / "r", {i: "" for i in range(len(REFERENCES))}) + scorer = _scorer(report_dir, dataset) + score, _ = scorer.score() + assert score == pytest.approx(0.0) + breakdown = scorer.score_breakdown() + assert breakdown is not None + assert breakdown["empty_responses"] == len(REFERENCES) + + def test_concatenated_words_are_credited(self, tmp_path: Path, dataset): + ds = MagicMock() + ds.dataframe = pd.DataFrame({"transcript": ["i can not go"]}) + ds.num_samples.return_value = 1 + report_dir = _stage(tmp_path / "r", {}) + (report_dir / "sample_idx_map.json").write_bytes( + msgspec.json.encode({DATASET_NAME: {"uuid-0": 0}}) + ) + encoder = msgspec.json.Encoder(enc_hook=EventType.encode_hook) + (report_dir / "events.jsonl").write_bytes( + encoder.encode( + EventRecord( + event_type=SampleEventType.COMPLETE, + sample_uuid="uuid-0", + data=TextModelOutput(output="I cannot go."), + ) + ) + + b"\n" + ) + assert _scorer(report_dir, ds).score()[0] == pytest.approx(100.0) + + +@pytest.mark.unit +@pytest.mark.usefixtures("hermetic_normalizer") +class TestPartialRuns: + def test_missing_sample_is_scored_as_an_empty_transcript( + self, tmp_path: Path, dataset + ): + # Only 2 of 3 issued samples completed, both perfectly. The third + # contributes its reference words as deletions rather than vanishing + # from the denominator, so the score must fall below 100. + report_dir = _stage(tmp_path / "r", {0: REFERENCES[0], 1: REFERENCES[1]}) + scorer = _scorer(report_dir, dataset) + score, _ = scorer.score() + breakdown = scorer.score_breakdown() + assert breakdown is not None + assert breakdown["missing_samples"] == 1 + # Counted through the same normalizer the scorer uses: it rewrites + # contractions, so a raw str.split() is not the reference word count. + assert breakdown["edit_distance"] == _ref_word_count(REFERENCES[2]) + assert breakdown["reference_words"] == sum( + _ref_word_count(r) for r in REFERENCES + ) + assert score == pytest.approx(100.0 - breakdown["wer_percent"]) + assert score < 100.0 + + def test_missing_samples_mark_the_score_incomplete(self, tmp_path: Path, dataset): + report_dir = _stage(tmp_path / "r", {0: REFERENCES[0], 1: REFERENCES[1]}) + scorer = _scorer(report_dir, dataset) + scorer.score() + assert scorer.complete is False + + def test_a_full_run_is_complete(self, tmp_path: Path, dataset): + report_dir = _stage(tmp_path / "r", dict(enumerate(REFERENCES))) + scorer = _scorer(report_dir, dataset) + scorer.score() + assert scorer.complete is True + + def test_an_empty_response_does_not_mark_the_run_incomplete( + self, tmp_path: Path, dataset + ): + # A blank transcript is a response: the sample completed and scored 0. + report_dir = _stage(tmp_path / "r", {i: "" for i in range(len(REFERENCES))}) + scorer = _scorer(report_dir, dataset) + scorer.score() + assert scorer.complete is True + breakdown = scorer.score_breakdown() + assert breakdown is not None + assert breakdown["missing_samples"] == 0 + + def test_no_completions_scores_zero_and_is_incomplete( + self, tmp_path: Path, dataset + ): + # Every issued sample is scored as an empty transcript, so the whole + # corpus is deletions: WER 100% -> score 0, not an absent number. + report_dir = _stage(tmp_path / "r", {}) + scorer = _scorer(report_dir, dataset) + score, _ = scorer.score() + assert score == pytest.approx(0.0) + assert scorer.complete is False + breakdown = scorer.score_breakdown() + assert breakdown is not None + assert breakdown["missing_samples"] == len(REFERENCES) + assert breakdown["wer_percent"] == pytest.approx(100.0) + + def test_nothing_issued_yields_no_score(self, tmp_path: Path, dataset): + report_dir = tmp_path / "r" + report_dir.mkdir() + (report_dir / "sample_idx_map.json").write_bytes( + msgspec.json.encode({DATASET_NAME: {}}) + ) + (report_dir / "events.jsonl").write_bytes(b"") + scorer = _scorer(report_dir, dataset) + assert scorer.score() == (None, 0) + assert scorer.complete is False + + +@pytest.mark.unit +@pytest.mark.usefixtures("hermetic_normalizer") +class TestContract: + def test_score_single_sample_is_not_usable(self, tmp_path: Path, dataset): + report_dir = _stage(tmp_path / "r", {0: REFERENCES[0]}) + with pytest.raises(RuntimeError, match="corpus-level"): + _scorer(report_dir, dataset).score_single_sample("a", "b") + + def test_breakdown_is_none_before_scoring(self, tmp_path: Path, dataset): + report_dir = _stage(tmp_path / "r", {0: REFERENCES[0]}) + assert _scorer(report_dir, dataset).score_breakdown() is None + + def test_default_normalizer_source(self, tmp_path: Path, dataset): + report_dir = _stage(tmp_path / "r", {}) + scorer = _scorer(report_dir, dataset) + assert scorer.normalizer_source == "openai/whisper-large-v3" + + def test_normalizer_source_override(self, tmp_path: Path, dataset): + report_dir = _stage(tmp_path / "r", {}) + scorer = _scorer(report_dir, dataset, normalizer_source="/local/whisper") + assert scorer.normalizer_source == "/local/whisper" + + +@pytest.mark.unit +class TestNormalizerResolution: + def test_tokenizer_without_spelling_map_is_rejected( + self, tmp_path: Path, dataset, monkeypatch: pytest.MonkeyPatch + ): + tokenizer = MagicMock(spec=[]) # no english_spelling_normalizer attribute + monkeypatch.setattr( + scoring.AutoTokenizer, "from_pretrained", lambda *a, **k: tokenizer + ) + report_dir = _stage(tmp_path / "r", {0: REFERENCES[0]}) + with pytest.raises(ValueError, match="english_spelling_normalizer"): + _scorer(report_dir, dataset)._normalizer() + + +@pytest.mark.unit +class TestPreflight: + """The normalizer is resolved before the run, not after the perf phase.""" + + def test_preflight_resolves_the_default_source( + self, monkeypatch: pytest.MonkeyPatch + ): + seen: list[str] = [] + + def _from_pretrained(source, *a, **k): + seen.append(source) + return MagicMock(english_spelling_normalizer={"a": "b"}) + + monkeypatch.setattr(scoring.AutoTokenizer, "from_pretrained", _from_pretrained) + WERScorer.preflight({}) + assert seen == [WERScorer.DEFAULT_NORMALIZER_SOURCE] + + def test_preflight_honours_the_extras_override( + self, monkeypatch: pytest.MonkeyPatch + ): + seen: list[str] = [] + + def _from_pretrained(source, *a, **k): + seen.append(source) + return MagicMock(english_spelling_normalizer={"a": "b"}) + + monkeypatch.setattr(scoring.AutoTokenizer, "from_pretrained", _from_pretrained) + WERScorer.preflight({"normalizer_source": "/local/whisper"}) + assert seen == ["/local/whisper"] + + def test_preflight_raises_on_an_unusable_tokenizer( + self, monkeypatch: pytest.MonkeyPatch + ): + # The whole point: this surfaces before any load is issued. + monkeypatch.setattr( + scoring.AutoTokenizer, + "from_pretrained", + lambda *a, **k: MagicMock(spec=[]), + ) + with pytest.raises(ValueError, match="english_spelling_normalizer"): + WERScorer.preflight({}) + + +@pytest.mark.unit +@pytest.mark.usefixtures("hermetic_normalizer") +class TestExtractorIsRejected: + def test_configured_extractor_is_an_error(self, tmp_path: Path, dataset): + # WER scores the transcript verbatim; an extractor would be ignored. + report_dir = _stage(tmp_path / "r", {0: REFERENCES[0]}) + with pytest.raises(InputValidationError, match="extractor"): + _scorer(report_dir, dataset, extractor=PythonCodeExtractor) + + def test_no_extractor_constructs(self, tmp_path: Path, dataset): + report_dir = _stage(tmp_path / "r", {0: REFERENCES[0]}) + assert _scorer(report_dir, dataset).extractor is None diff --git a/tests/unit/openai/test_transcriptions_adapter.py b/tests/unit/openai/test_transcriptions_adapter.py new file mode 100644 index 000000000..710d85909 --- /dev/null +++ b/tests/unit/openai/test_transcriptions_adapter.py @@ -0,0 +1,222 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for OpenAITranscriptionsAdapter.""" + +from pathlib import Path + +import msgspec +import pandas as pd +import pytest +from inference_endpoint.config.schema import ModelParams +from inference_endpoint.core.types import Query +from inference_endpoint.dataset_manager.transforms import apply_transforms +from inference_endpoint.openai.transcriptions_adapter import ( + OpenAITranscriptionsAdapter as Adapter, +) + +WAV_BYTES = b"RIFF\x24\x00\x00\x00WAVEfmt " + bytes(range(256)) + + +@pytest.fixture +def audio_file(tmp_path: Path) -> Path: + path = tmp_path / "clip-0001.wav" + path.write_bytes(WAV_BYTES) + return path + + +def _query(audio_file: Path, **extra) -> Query: + data = { + "audio_file": str(audio_file), + "model": "openai/whisper-large-v3", + "language": "en", + "response_format": "json", + "temperature": 0.0, + } + data.update(extra) + return Query(id="q1", data=data) + + +@pytest.mark.unit +class TestContentType: + def test_content_type_is_multipart_with_boundary(self): + assert Adapter.CONTENT_TYPE.startswith("multipart/form-data; boundary=") + + def test_declared_boundary_matches_every_encoded_body(self, audio_file: Path): + # The worker folds Content-Type into a prefix built once at setup, so a + # body whose boundary differed from the declared one would be unparseable. + boundary = Adapter.CONTENT_TYPE.split("boundary=", 1)[1].encode() + for _ in range(2): + body = Adapter.encode_query(_query(audio_file)) + assert body.startswith(b"--" + boundary + b"\r\n") + + +@pytest.mark.unit +class TestEncodeQuery: + def test_body_contains_audio_bytes_verbatim(self, audio_file: Path): + body = Adapter.encode_query(_query(audio_file)) + assert WAV_BYTES in body + + def test_body_carries_filename_and_part_content_type(self, audio_file: Path): + body = Adapter.encode_query(_query(audio_file)) + assert b'name="file"; filename="clip-0001.wav"' in body + assert b"Content-Type: audio/wav" in body + + def test_body_carries_form_fields(self, audio_file: Path): + body = Adapter.encode_query(_query(audio_file)) + for name, value in ( + ("model", b"openai/whisper-large-v3"), + ("language", b"en"), + ("response_format", b"json"), + ("temperature", b"0.0"), + ): + assert f'name="{name}"'.encode() in body + assert value in body + + def test_body_is_closed_with_terminating_boundary(self, audio_file: Path): + boundary = Adapter.CONTENT_TYPE.split("boundary=", 1)[1] + body = Adapter.encode_query(_query(audio_file)) + assert body.endswith(f"--{boundary}--\r\n".encode()) + + def test_absent_optional_fields_are_omitted(self, audio_file: Path): + query = Query(id="q", data={"audio_file": str(audio_file), "model": "m"}) + body = Adapter.encode_query(query) + assert b'name="language"' not in body + assert b'name="temperature"' not in body + + def test_none_valued_fields_are_omitted(self, audio_file: Path): + # AddStaticColumns materialises model_params defaults that may be None; + # sending "None" as a form value would be rejected by the server. + query = _query(audio_file, temperature=None) + assert b'name="temperature"' not in Adapter.encode_query(query) + + def test_missing_audio_file_raises_keyerror(self): + with pytest.raises(KeyError, match="audio_file"): + Adapter.encode_query(Query(id="q", data={"model": "m"})) + + def test_missing_audio_on_disk_raises(self, tmp_path: Path): + query = Query( + id="q", data={"audio_file": str(tmp_path / "gone.wav"), "model": "m"} + ) + with pytest.raises(FileNotFoundError): + Adapter.encode_query(query) + + +@pytest.mark.unit +class TestBodyInjection: + """Dataset-controlled strings must not be able to forge headers or parts.""" + + def test_quote_in_filename_is_escaped(self, tmp_path: Path): + path = tmp_path / 'ev"il.wav' + path.write_bytes(WAV_BYTES) + body = Adapter.encode_query(_query(path)) + # The raw quote would close filename= early; %22 is what servers decode. + assert b'filename="ev%22il.wav"' in body + assert b'filename="ev"il.wav"' not in body + + def test_crlf_in_filename_is_escaped(self, tmp_path: Path): + # A bare CRLF would end the Content-Disposition line and let the rest of + # the name be read as its own header. + path = tmp_path / "a\r\nX-Injected: yes\r\n\r\nx.wav" + path.write_bytes(WAV_BYTES) + body = Adapter.encode_query(_query(path)) + assert b"%0D%0AX-Injected: yes" in body + assert b"\r\nX-Injected: yes\r\n\r\nx.wav" not in body + + def test_field_value_carrying_the_delimiter_is_rejected(self, audio_file: Path): + boundary = Adapter.CONTENT_TYPE.split("boundary=", 1)[1] + query = _query(audio_file, prompt=f"x\r\n--{boundary}\r\nforged") + with pytest.raises(ValueError, match="delimiter"): + Adapter.encode_query(query) + + def test_ordinary_multiline_field_value_is_allowed(self, audio_file: Path): + # CRLF is legal inside a part body; only the delimiter is a forgery. + query = _query(audio_file, prompt="line one\r\nline two") + assert b"line one\r\nline two" in Adapter.encode_query(query) + + +@pytest.mark.unit +class TestDecodeResponse: + def test_decodes_text_field(self): + result = Adapter.decode_response(b'{"text": "hello world"}', "q1") + assert result.id == "q1" + assert result.error is None + assert str(result.response_output) == "hello world" + + def test_ignores_server_specific_extras(self): + body = b'{"text": "hi", "duration": 29.9, "language": "english"}' + assert str(Adapter.decode_response(body, "q").response_output) == "hi" + + def test_missing_text_raises(self): + with pytest.raises(msgspec.ValidationError): + Adapter.decode_response(b'{"duration": 1.0}', "q") + + def test_malformed_json_raises(self): + with pytest.raises(msgspec.DecodeError): + Adapter.decode_response(b"not json", "q") + + +@pytest.mark.unit +class TestSse: + def test_decode_sse_message_raises(self): + with pytest.raises(NotImplementedError): + Adapter.decode_sse_message(b'{"text": "x"}') + + +@pytest.mark.unit +class TestDatasetTransforms: + def _apply(self, df: pd.DataFrame, **model_kwargs) -> pd.DataFrame: + params = ModelParams(name="openai/whisper-large-v3", **model_kwargs) + return apply_transforms(df, Adapter.dataset_transforms(params)) + + def test_drops_ground_truth_columns_from_the_request_row(self): + df = pd.DataFrame( + [{"audio_file": "/a.wav", "transcript": "hello", "duration": 29.5}] + ) + out = self._apply(df) + assert "transcript" not in out.columns + assert "duration" not in out.columns + assert out.iloc[0]["audio_file"] == "/a.wav" + + def test_adds_model_and_response_format(self): + df = pd.DataFrame([{"audio_file": "/a.wav", "transcript": "hello"}]) + row = self._apply(df, temperature=0.0).iloc[0] + assert row["model"] == "openai/whisper-large-v3" + assert row["response_format"] == "json" + assert row["temperature"] == 0.0 + + def test_language_is_sent_by_default(self): + # Without it the server auto-detects per clip, which is a different + # workload from the MLPerf reference's pinned-language decode. + df = pd.DataFrame([{"audio_file": "/a.wav", "transcript": "hello"}]) + assert self._apply(df).iloc[0]["language"] == "en" + + def test_default_language_reaches_the_request_body(self, audio_file: Path): + df = pd.DataFrame([{"audio_file": str(audio_file), "transcript": "hello"}]) + row = self._apply(df).iloc[0] + body = Adapter.encode_query(Query(id="q", data=row.to_dict())) + assert b'name="language"' in body + assert b"\r\n\r\nen\r\n" in body + + def test_dataset_language_overrides_defaults(self): + df = pd.DataFrame( + [{"audio_file": "/a.wav", "transcript": "hola", "language": "es"}] + ) + assert self._apply(df).iloc[0]["language"] == "es" + + def test_missing_audio_file_column_raises(self): + df = pd.DataFrame([{"transcript": "hello"}]) + with pytest.raises(KeyError, match="audio_file"): + self._apply(df) diff --git a/tests/unit/openai/test_transcriptions_registration.py b/tests/unit/openai/test_transcriptions_registration.py new file mode 100644 index 000000000..92546784f --- /dev/null +++ b/tests/unit/openai/test_transcriptions_registration.py @@ -0,0 +1,75 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests that APIType.OPENAI_TRANSCRIPTIONS is registered and wired up correctly.""" + +from importlib import import_module + +import pytest +from inference_endpoint.core.types import APIType +from inference_endpoint.endpoint_client.config import ( + ACCUMULATOR_MAP, + ADAPTER_MAP, + HTTPClientConfig, +) + + +def _resolve(path: str) -> type: + module_path, class_name = path.rsplit(".", 1) + return getattr(import_module(module_path), class_name) + + +@pytest.mark.unit +def test_api_type_exists(): + assert APIType.OPENAI_TRANSCRIPTIONS == "openai_transcriptions" + + +@pytest.mark.unit +def test_default_route(): + assert APIType.OPENAI_TRANSCRIPTIONS.default_route() == "v1/audio/transcriptions" + + +@pytest.mark.unit +def test_in_adapter_map(): + path = ADAPTER_MAP[APIType.OPENAI_TRANSCRIPTIONS] + assert _resolve(path).__name__ == "OpenAITranscriptionsAdapter" + + +@pytest.mark.unit +def test_in_accumulator_map(): + # An omitted entry silently falls back to the OpenAI SSE accumulator, which + # would surface zero-output queries as successful on a non-streaming route. + path = ACCUMULATOR_MAP[APIType.OPENAI_TRANSCRIPTIONS] + assert _resolve(path).__name__ == "NonStreamingAccumulator" + + +@pytest.mark.unit +def test_client_config_resolves_the_pair(): + config = HTTPClientConfig(api_type=APIType.OPENAI_TRANSCRIPTIONS) + assert config.adapter.__name__ == "OpenAITranscriptionsAdapter" + assert config.accumulator.__name__ == "NonStreamingAccumulator" + + +@pytest.mark.unit +def test_adapter_declares_multipart_content_type(): + adapter = _resolve(ADAPTER_MAP[APIType.OPENAI_TRANSCRIPTIONS]) + assert adapter.CONTENT_TYPE.startswith("multipart/form-data; boundary=") + + +@pytest.mark.unit +def test_other_adapters_still_declare_json(): + for api_type in (APIType.OPENAI, APIType.OPENAI_COMPLETIONS, APIType.SGLANG): + adapter = _resolve(ADAPTER_MAP[api_type]) + assert adapter.CONTENT_TYPE == "application/json"