diff --git a/.gitignore b/.gitignore index fdc44128c..c456be335 100644 --- a/.gitignore +++ b/.gitignore @@ -150,6 +150,9 @@ plans/ # Benchmark result directories bench_results*/ +soak-results/ +local-soak-test-results/ +routing-benchmark-results/ jobs/**/*.* pytest-of-*/ diff --git a/Cargo.lock b/Cargo.lock index 405dda9da..ff7a91185 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2388,6 +2388,23 @@ dependencies = [ "tokio", ] +[[package]] +name = "switchyard-soak" +version = "0.2.0" +dependencies = [ + "axum", + "clap", + "futures-util", + "humantime", + "parking_lot", + "rand 0.10.2", + "reqwest", + "serde", + "serde_json", + "tempfile", + "tokio", +] + [[package]] name = "switchyard-translation" version = "0.2.0" diff --git a/Cargo.toml b/Cargo.toml index 98433d8c1..d929d0c58 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,6 +10,7 @@ members = [ "crates/protocol", "crates/switchyard-server", "crates/switchyard-skill-distillation", + "crates/switchyard-soak", "crates/switchyard-translation", ] diff --git a/crates/libsy/src/algorithms/noop.rs b/crates/libsy/src/algorithms/noop.rs index 95f4c7212..8a9e42c47 100644 --- a/crates/libsy/src/algorithms/noop.rs +++ b/crates/libsy/src/algorithms/noop.rs @@ -23,11 +23,12 @@ impl Algorithm for Noop { } async fn route(self: Arc, _driver: Driver, request: Request) -> Result { + let streaming = request.llm_request.stream; let model_id = request .model_id() .unwrap_or_else(|| ModelId::from("switchyard/noop")); tracing::info!(target = %model_id, "noop returned its synthetic response"); - let llm_response = LlmResponse::Agg(AggLlmResponse { + let aggregate = AggLlmResponse { id: Some("switchyard-noop".to_string()), model: Some(model_id.to_string()), outputs: vec![ResponseOutput { @@ -38,7 +39,12 @@ impl Algorithm for Noop { stop_reason: Some(StopReason::EndTurn), }], ..Default::default() - }); + }; + let llm_response = if streaming { + LlmResponse::Stream(aggregate.into_stream()) + } else { + LlmResponse::Agg(aggregate) + }; let response = Response { llm_response, metadata: request.metadata.clone(), @@ -75,4 +81,24 @@ mod tests { assert_eq!(response.selected_model(), Some(TEST_MODEL)); Ok(()) } + + #[tokio::test] + async fn test_noop_algo_streams_when_requested() -> Result<()> { + let request = Request { + llm_request: LlmRequest { + model: Some("streaming-noop".to_string()), + messages: vec![Message::text(Role::User, "hi")], + stream: true, + ..LlmRequest::default() + }, + raw_request: None, + metadata: None, + }; + + let algorithm: Arc = Arc::new(Noop {}); + let (_, response) = test_drive(algorithm, request, echo()).await?; + + assert!(matches!(response.llm_response, LlmResponse::Stream(_))); + Ok(()) + } } diff --git a/crates/switchyard-server/src/config.rs b/crates/switchyard-server/src/config.rs index a78695cd5..ea5e86d43 100644 --- a/crates/switchyard-server/src/config.rs +++ b/crates/switchyard-server/src/config.rs @@ -1652,6 +1652,21 @@ target = "azure" return Err(ServerError::new("primary llm client is missing")); }; assert_eq!(primary.max_retries, 0); + + let maximum = VALID_CONFIG.replacen( + "base_url = \"https://example.test/v1\"", + &format!( + "base_url = \"https://example.test/v1\"\nmax_retries = {MAX_CONFIGURED_RETRIES}" + ), + 1, + ); + let config: ServerConfig = toml::from_str(&maximum).map_err(|error| { + ServerError::new(format!("failed to parse maximum retry config: {error}")) + })?; + let Some(primary) = config.llm_clients.get("primary") else { + return Err(ServerError::new("primary llm client is missing")); + }; + assert_eq!(primary.max_retries, MAX_CONFIGURED_RETRIES); Ok(()) } diff --git a/crates/switchyard-soak/Cargo.toml b/crates/switchyard-soak/Cargo.toml new file mode 100644 index 000000000..242dbed17 --- /dev/null +++ b/crates/switchyard-soak/Cargo.toml @@ -0,0 +1,37 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "switchyard-soak" +version.workspace = true +description = "Sustained load tester for a live Switchyard server" +authors.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +readme = "README.md" + +[[bin]] +name = "switchyard-soak" +path = "src/main.rs" + +[[example]] +name = "switchyard-soak-mock" +path = "examples/mock_server.rs" + +[dependencies] +clap = { version = "4", features = ["derive"] } +futures-util.workspace = true +humantime = "2.4" +parking_lot.workspace = true +rand.workspace = true +reqwest.workspace = true +serde.workspace = true +serde_json.workspace = true +tokio.workspace = true + +[dev-dependencies] +axum = "0.8" +tempfile = "3" +tokio.workspace = true diff --git a/crates/switchyard-soak/README.md b/crates/switchyard-soak/README.md new file mode 100644 index 000000000..7f739f52f --- /dev/null +++ b/crates/switchyard-soak/README.md @@ -0,0 +1,119 @@ +# switchyard-soak + +`switchyard-soak` keeps a fixed number of requests in flight against one route on a live +`switchyard-server`. Its scenario catalog covers short and long inputs, long outputs, shared +prefixes, mixed traffic, growing conversations, tool schemas, tool-call bursts, routing signals, +and bounded failure cases. The short baseline also exercises Chat Completions, Messages, and +Responses in streaming and non-streaming form. The command samples health, metrics, and an +optional local server process, then exits with status 1 when a release gate fails. + +Build the server and soak tester from the commit being tested: + +```bash +cargo build --release -p switchyard-server -p switchyard-soak \ + --bins --example switchyard-soak-mock +``` + +Run a five-minute check against a route advertised by `GET /v1/models`: + +```bash +target/release/switchyard-soak \ + --base-url http://127.0.0.1:4000 \ + --model switchyard/general \ + --duration 5m \ + --concurrency 4 \ + --report-interval 10 +``` + +Run the 48-hour release gate and measure the local server process: + +```bash +target/release/switchyard-soak \ + --model switchyard/general \ + --duration 48h \ + --concurrency 16 \ + --server-pid "$SWITCHYARD_SERVER_PID" \ + --max-rss-growth-mib 512 +``` + +If the endpoint needs a bearer token, put the secret in an environment variable and pass the +variable's name: + +```bash +export SWITCHYARD_SOAK_TOKEN="..." +target/release/switchyard-soak \ + --model switchyard/general \ + --api-key-env SWITCHYARD_SOAK_TOKEN +``` + +## Flags + +| Flag | Default | What it does | +|---|---:|---| +| `--base-url URL` | `http://127.0.0.1:4000` | Sends every check and inference request to this Switchyard server. | +| `--model ID` | required | Selects one exact route id returned by `GET /v1/models`. The test stops before sending load if the id is missing. | +| `--duration TIME` | `48h` | Keeps the load running for this long. Use seconds, minutes, or hours, such as `30s`, `5m`, or `48h`. | +| `--concurrency N` | `16` | Keeps this many requests in flight. Each worker waits for its response before sending another request. | +| `--max-output-tokens N` | `32` | Sets the default output limit. `decode-heavy` uses its bounded 512-token and 1,024-token limits. | +| `--prompt-bytes N` | `1024` | Sizes the repeated input used by short and prefix-reuse scenarios. Larger values put more pressure on request memory and prefix caching. | +| `--scenario-set NAME` | `standard` | Selects `core`, `agentic`, `resilience`, `standard`, or `all`. `standard` includes core and agentic scenarios without expected failures. | +| `--scenario ID` | none | Selects one exact scenario id. Repeat the flag to select more scenarios; explicit ids take precedence over `--scenario-set`. | +| `--context-window-tokens N` | `32768` | Bounds generated long-context and overflow inputs. The command rejects values above 1,000,000. | +| `--export-scenarios PATH` | none | Writes a manifest and AIPerf `inputs-json` files, then exits without contacting a server. The performance script uses this mode. | +| `--request-timeout SECONDS` | `120` | Limits connection time and idle time between response bytes. A healthy stream may run longer if it keeps producing data. | +| `--report-interval SECONDS` | `60` | Sets how often the test samples health, metrics, RSS, CPU, and interval latency. | +| `--invalid-canary-interval SECONDS` | `300` | Sends malformed input this often, expects HTTP 400, and then checks recovery. Set it to `0` only when a permissive mock cannot reject the canary. | +| `--max-error-rate FRACTION` | `0` | Allows at most this fraction of inference requests to fail. `0.01` means one percent. The value must be between 0 and 1. | +| `--server-pid PID` | none | Samples RSS and CPU from this local `switchyard-server` process. Omit it for a remote server. | +| `--max-rss-growth-mib MIB` | none | Fails when the last RSS sample exceeds the first by more than this amount. It requires `--server-pid`. | +| `--api-key-env NAME` | none | Reads a bearer token from this environment variable without putting the secret in the command or result files. | +| `--results-dir PATH` | `soak-results/` | Creates this new directory for the run. The command refuses to reuse an existing directory. | +| `--help` | n/a | Prints the command reference and examples. | +| `--version` | n/a | Prints the crate version. | + +The scenario source lives in `src/scenarios/`. Each request shape has one file and a pure builder. +The exported manifest is the only scenario input read by the Python performance runner, so model +ids, sessions, tools, and message histories cannot drift between the Rust soak and AIPerf runs. +Concurrency knees and traffic bursts are load profiles attached to the short baseline, not +duplicate request scenarios. + +The catalog contains: + +| Group | Scenarios | +|---|---| +| Core | `short-interactive`, `long-context`, `decode-heavy`, `prefix-reuse`, `mixed-traffic` | +| Agentic | `growing-conversation`, `large-tool-catalog`, `tool-call-burst`, `stage-transitions`, `classifier-mix` | +| Resilience | `context-overflow`, `failure-pressure`, `client-cancellation` | + +## Check the local server and load tools + +`scripts/run_local_soak_test.py` starts the crate's request-aware scenario backend and +`switchyard-server`, sends one HTTP request through each configured route, then runs `oha` and +NVIDIA AIPerf sequentially for every routing algorithm. The backend returns valid classifier +verdicts and injects the context, retry, stream, and cancellation cases named by resilience +scenarios. It resets transient-failure counters before each AIPerf cell so every routing algorithm +receives the same attempts. For streaming requests it emits the scenario's requested output length +with a configurable delay between tokens, which keeps local TTFT, ITL, and token-throughput +comparisons deterministic. The local test needs no provider key and incurs no inference cost. + +Run `scripts/benchmark_routing_algorithms.py` by itself against an existing Switchyard server to +compare the same routes with real model output. oha runs only for the fixed `short-interactive` +baseline. AIPerf replays every selected session and reports TTFT, ITL, request throughput, output +token throughput, multi-run confidence, selected-target calls, classifier calls, and routing +overhead. Add `--direct-base-url` and `--direct-model` to compare each route with the same backend +without Switchyard. That comparison adds an annotated TTFT and output-token-throughput SVG to the +Markdown report. The operations guide explains how to keep the comparison fair and when to use the +local backend or real models. + +The script needs the built Rust programs plus installed `oha` and AIPerf commands. +`switchyard-soak` itself does not require those extra programs and can run by itself against a +release server. See the [operations guide](../../docs/operations/soak_test.md) for setup, the local +command, the 48-hour release run, and result review. + +## Results + +Each run creates `config.json`, `intervals.csv`, `errors.jsonl`, and `summary.json`. Error details +stop at 10,000 records, and run-wide latency uses a 100,000-sample reservoir, so failures and a +high request rate cannot grow memory or disk use without bound. + +See `docs/operations/soak_test.md` for release preparation, pass criteria, and result review. diff --git a/crates/switchyard-soak/examples/mock_server.rs b/crates/switchyard-soak/examples/mock_server.rs new file mode 100644 index 000000000..c92578fd6 --- /dev/null +++ b/crates/switchyard-soak/examples/mock_server.rs @@ -0,0 +1,280 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Request-aware local backend used by `scripts/run_local_soak_test.py`. + +use std::collections::HashMap; +use std::convert::Infallible; +use std::process::ExitCode; +use std::sync::Arc; +use std::time::Duration; + +use axum::Router; +use axum::extract::{Json, State}; +use axum::http::StatusCode; +use axum::response::sse::{Event, Sse}; +use axum::response::{IntoResponse, Response}; +use axum::routing::{get, post}; +use clap::Parser; +use futures_util::{StreamExt, stream}; +use parking_lot::Mutex; +use serde_json::{Value, json}; + +#[derive(Parser)] +#[command( + name = "switchyard-soak-mock", + about = "Start the request-aware backend for the local Switchyard scenario test", + after_long_help = "Example:\n cargo run --release -p switchyard-soak --example switchyard-soak-mock -- --port 8100 --latency-ms 40", + version +)] +struct Args { + /// Local TCP port used by the backend. + #[arg(long, default_value_t = 8100)] + port: u16, + + /// Artificial delay, in milliseconds, added to ordinary responses. + #[arg(long, default_value_t = 40)] + latency_ms: u64, + + /// Artificial delay between streamed output tokens, in milliseconds. + #[arg(long, default_value_t = 1)] + token_latency_ms: u64, +} + +#[derive(Clone)] +struct BackendState { + latency: Duration, + token_latency: Duration, + attempts: Arc>>, +} + +// AIPerf requires this OpenAI field; a fixed value keeps local runs deterministic. +const FIXED_CREATED_AT: u64 = 1_700_000_000; + +fn scenario_marker(body: &Value) -> Option<&str> { + body.get("messages")? + .as_array()? + .iter() + .filter_map(|message| message.get("content").and_then(Value::as_str)) + .find_map(|content| { + let marker = content.split_once("[scenario:")?.1; + marker.split_once(']').map(|(name, _rest)| name) + }) +} + +fn classifier_content(marker: Option<&str>) -> &'static str { + if marker == Some("classifier_invalid") { + return "not a JSON verdict"; + } + if marker == Some("classifier_hard") { + return r#"{"crux":"distributed diagnosis","primary_rule":"LIM-1","capability_boundary":"unsupported","p_solve":0.1}"#; + } + r#"{"crux":"bounded task","primary_rule":"SUP-1","capability_boundary":"supported","p_solve":1.0}"# +} + +fn completion(model: &str, content: &str) -> Value { + json!({ + "id": "chatcmpl-switchyard-soak", + "object": "chat.completion", + "created": FIXED_CREATED_AT, + "model": model, + "choices": [{ + "index": 0, + "message": {"role": "assistant", "content": content}, + "finish_reason": "stop" + }], + "usage": {"prompt_tokens": 32, "completion_tokens": 2, "total_tokens": 34} + }) +} + +fn requested_output_tokens(body: &Value) -> u64 { + body.get("max_completion_tokens") + .or_else(|| body.get("max_tokens")) + .and_then(Value::as_u64) + .unwrap_or(2) + .clamp(1, 4_096) +} + +fn stream( + model: &str, + truncated: bool, + completion_tokens: u64, + token_latency: Duration, +) -> Response { + let mut events = Vec::with_capacity(completion_tokens as usize + 2); + for index in 0..completion_tokens { + events.push( + json!({ + "id": "chatcmpl-switchyard-soak", + "object": "chat.completion.chunk", + "created": FIXED_CREATED_AT, + "model": model, + "choices": [{ + "index": 0, + "delta": { + "role": (index == 0).then_some("assistant"), + "content": if index == 0 { "token" } else { " token" } + }, + "finish_reason": null + }] + }) + .to_string(), + ); + if truncated { + break; + } + } + if !truncated { + events.push( + json!({ + "id": "chatcmpl-switchyard-soak", + "object": "chat.completion.chunk", + "created": FIXED_CREATED_AT, + "model": model, + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], + "usage": { + "prompt_tokens": 32, + "completion_tokens": completion_tokens, + "total_tokens": 32 + completion_tokens + } + }) + .to_string(), + ); + events.push("[DONE]".to_string()); + } + let events = + stream::iter(events.into_iter().enumerate()).then(move |(index, data)| async move { + if index > 0 { + tokio::time::sleep(token_latency).await; + } + Ok::(Event::default().data(data)) + }); + Sse::new(events).into_response() +} + +async fn chat(State(state): State, Json(body): Json) -> Response { + let model = body + .get("model") + .and_then(Value::as_str) + .unwrap_or_default(); + let marker = scenario_marker(&body); + if marker == Some("client_cancellation") { + tokio::time::sleep(Duration::from_secs(2)).await; + } else { + tokio::time::sleep(state.latency).await; + } + + if model == "mock/classifier" { + return Json(completion(model, classifier_content(marker))).into_response(); + } + if model == "mock/weak" && marker == Some("context_overflow") { + return ( + StatusCode::BAD_REQUEST, + Json(json!({ + "error": { + "code": "context_length_exceeded", + "message": "the weak target context window is too small" + } + })), + ) + .into_response(); + } + + for (name, status) in [ + ("upstream_429", StatusCode::TOO_MANY_REQUESTS), + ("upstream_500", StatusCode::INTERNAL_SERVER_ERROR), + ] { + if marker == Some(name) { + let key = format!("{model}:{name}"); + let attempt = { + let mut attempts = state.attempts.lock(); + let attempt = attempts.entry(key).or_default(); + *attempt += 1; + *attempt + }; + if attempt <= 2 { + return ( + status, + Json( + json!({"error": {"message": format!("injected {name} attempt {attempt}")}}), + ), + ) + .into_response(); + } + } + } + + if body.get("stream") == Some(&Value::Bool(true)) { + return stream( + model, + marker == Some("truncated_stream"), + requested_output_tokens(&body), + state.token_latency, + ); + } + Json(completion(model, "OK")).into_response() +} + +async fn run(args: Args) -> Result<(), String> { + if args.port == 0 { + return Err("--port must be greater than zero".to_string()); + } + let listener = tokio::net::TcpListener::bind(("127.0.0.1", args.port)) + .await + .map_err(|error| error.to_string())?; + let address = listener.local_addr().map_err(|error| error.to_string())?; + let state = BackendState { + latency: Duration::from_millis(args.latency_ms), + token_latency: Duration::from_millis(args.token_latency_ms), + attempts: Arc::new(Mutex::new(HashMap::new())), + }; + let app = Router::new() + .route("/health", get(|| async { Json(json!({"status": "ok"})) })) + .route( + "/reset", + post(|State(state): State| async move { + state.attempts.lock().clear(); + Json(json!({"status": "reset"})) + }), + ) + .route("/v1/chat/completions", post(chat)) + .with_state(state); + println!("Scenario backend is ready at http://{address}"); + axum::serve(listener, app) + .with_graceful_shutdown(async { + if let Err(error) = tokio::signal::ctrl_c().await { + eprintln!("could not register the shutdown signal: {error}"); + std::future::pending::<()>().await; + } + }) + .await + .map_err(|error| error.to_string()) +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::requested_output_tokens; + + #[test] + fn output_tokens_follow_openai_limits_and_stay_bounded() { + assert_eq!(requested_output_tokens(&json!({"max_tokens": 512})), 512); + assert_eq!( + requested_output_tokens(&json!({"max_completion_tokens": 8_192})), + 4_096 + ); + assert_eq!(requested_output_tokens(&json!({})), 2); + } +} + +#[tokio::main] +async fn main() -> ExitCode { + match run(Args::parse()).await { + Ok(()) => ExitCode::SUCCESS, + Err(error) => { + eprintln!("switchyard-soak-mock failed: {error}"); + ExitCode::FAILURE + } + } +} diff --git a/crates/switchyard-soak/src/client.rs b/crates/switchyard-soak/src/client.rs new file mode 100644 index 000000000..5fe7074b1 --- /dev/null +++ b/crates/switchyard-soak/src/client.rs @@ -0,0 +1,525 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! HTTP against the public Switchyard APIs: preflight, one request, and server-state reads. + +use futures_util::StreamExt; +use reqwest::Client; +use reqwest::header::CONTENT_TYPE; +use serde_json::{Value, json}; + +const SERVER_REQUESTS_METRIC: &str = "switchyard_total_requests"; +const SERVER_ERRORS_METRIC: &str = "switchyard_total_errors"; + +/// One public Switchyard API the soak test exercises. +#[derive(Clone, Copy, PartialEq, Eq)] +pub enum Endpoint { + Chat, + Messages, + Responses, +} + +impl Endpoint { + pub const ALL: [Endpoint; 3] = [Endpoint::Chat, Endpoint::Messages, Endpoint::Responses]; + + pub fn path(self) -> &'static str { + match self { + Endpoint::Chat => "/v1/chat/completions", + Endpoint::Messages => "/v1/messages", + Endpoint::Responses => "/v1/responses", + } + } + + /// Field a successful response for this endpoint must contain. + fn required_field(self) -> &'static str { + match self { + Endpoint::Chat => "choices", + Endpoint::Messages => "content", + Endpoint::Responses => "output", + } + } + + pub fn as_str(self) -> &'static str { + match self { + Endpoint::Chat => "chat", + Endpoint::Messages => "messages", + Endpoint::Responses => "responses", + } + } +} + +/// Keep at most *limit* characters, on a char boundary, for a logged detail string. +fn truncate(text: &str, limit: usize) -> String { + text.chars().take(limit).collect() +} + +fn transport_error_kind(error: &reqwest::Error) -> &'static str { + if error.is_timeout() { + "timeout" + } else if error.is_decode() { + "request_error" + } else if error.is_connect() || error.is_request() || error.is_body() { + "transport" + } else { + "request_error" + } +} + +/// Build one request body for a public Switchyard API. +pub fn request_body( + endpoint: Endpoint, + model: &str, + prompt: &str, + max_output_tokens: u32, + stream: bool, +) -> Value { + match endpoint { + // Chat Completions and Anthropic Messages take the same model/messages/max_tokens body. + Endpoint::Chat | Endpoint::Messages => json!({ + "model": model, + "messages": [{"role": "user", "content": prompt}], + "max_tokens": max_output_tokens, + "temperature": 0, + "stream": stream, + }), + Endpoint::Responses => json!({ + "model": model, + "input": prompt, + "max_output_tokens": max_output_tokens, + "stream": stream, + }), + } +} + +#[derive(Default)] +struct Metrics { + requests: Option, + errors: Option, +} + +fn parse_metrics(text: &str) -> Metrics { + let mut metrics = Metrics::default(); + for line in text.lines() { + if line.is_empty() || line.starts_with('#') { + continue; + } + let Some((name_part, rest)) = line.split_once(' ') else { + continue; + }; + let name = name_part.split('{').next().unwrap_or(name_part); + if let Some(token) = rest.split_whitespace().next() + && let Ok(value) = token.parse::() + { + match name { + SERVER_REQUESTS_METRIC => metrics.requests = Some(value), + SERVER_ERRORS_METRIC => metrics.errors = Some(value), + _ => {} + } + } + } + metrics +} + +#[derive(Debug)] +pub struct RequestError { + pub kind: String, + pub detail: String, +} + +impl RequestError { + fn new(kind: impl Into, detail: impl Into) -> Self { + Self { + kind: kind.into(), + detail: detail.into(), + } + } + + fn transport(error: &reqwest::Error) -> Self { + Self::new( + transport_error_kind(error), + truncate(&error.to_string(), 500), + ) + } +} + +struct StreamValidator { + endpoint: Endpoint, + event_name: Option, + saw_json: bool, + saw_terminal: bool, +} + +impl StreamValidator { + fn new(endpoint: Endpoint) -> Self { + Self { + endpoint, + event_name: None, + saw_json: false, + saw_terminal: false, + } + } + + fn read_line(&mut self, line: &[u8]) -> Result<(), RequestError> { + let line = line.strip_suffix(b"\r").unwrap_or(line); + if line.is_empty() { + self.event_name = None; + return Ok(()); + } + let line = std::str::from_utf8(line) + .map_err(|error| RequestError::new("invalid_stream", error.to_string()))?; + if let Some(event_name) = line.strip_prefix("event:") { + self.event_name = Some(event_name.trim_start().to_string()); + return Ok(()); + } + let Some(data) = line.strip_prefix("data:") else { + return Ok(()); + }; + let data = data.strip_prefix(' ').unwrap_or(data); + if self.saw_terminal { + return Err(RequestError::new( + "invalid_stream", + "stream contained data after its terminal event", + )); + } + if data == "[DONE]" { + if self.endpoint != Endpoint::Chat { + return Err(RequestError::new( + "invalid_stream", + format!( + "{} stream contained an OpenAI [DONE] marker", + self.endpoint.as_str() + ), + )); + } + self.saw_terminal = true; + return Ok(()); + } + + let payload: Value = serde_json::from_str(data).map_err(|error| { + RequestError::new("invalid_stream", format!("invalid SSE JSON: {error}")) + })?; + let event_type = payload.get("type").and_then(Value::as_str); + if self.event_name.as_deref() == Some("error") + || event_type == Some("error") + || payload.get("error").is_some() + { + let detail = payload + .pointer("/error/message") + .or_else(|| payload.get("message")) + .and_then(Value::as_str) + .unwrap_or("stream returned an error event"); + return Err(RequestError::new("stream_error", truncate(detail, 500))); + } + + match self.endpoint { + Endpoint::Chat => {} + Endpoint::Messages | Endpoint::Responses => { + let event_type = event_type.ok_or_else(|| { + RequestError::new("invalid_stream", "SSE data did not contain an event type") + })?; + if self.event_name.as_deref() != Some(event_type) { + return Err(RequestError::new( + "invalid_stream", + format!( + "SSE event name {:?} did not match data type {event_type:?}", + self.event_name + ), + )); + } + self.saw_terminal = matches!( + (self.endpoint, event_type), + (Endpoint::Messages, "message_stop") + | ( + Endpoint::Responses, + "response.completed" | "response.incomplete" + ) + ); + } + } + self.saw_json = true; + Ok(()) + } + + fn finish(self) -> Result<(), RequestError> { + if !self.saw_json { + return Err(RequestError::new( + "empty_stream", + "successful streaming response contained no JSON events", + )); + } + if !self.saw_terminal { + return Err(RequestError::new( + "invalid_stream", + format!( + "{} stream ended without its terminal event", + self.endpoint.as_str() + ), + )); + } + Ok(()) + } +} + +/// Send one request and validate its response body or event stream. +pub async fn send_request( + client: &Client, + base_url: &str, + endpoint: Endpoint, + session_id: &str, + body: &Value, +) -> Result<(), RequestError> { + let url = format!("{base_url}{}", endpoint.path()); + let stream = body.get("stream").and_then(Value::as_bool).unwrap_or(false); + let response = match client + .post(&url) + .header("x-switchyard-session-id", session_id) + .json(body) + .send() + .await + { + Ok(response) => response, + Err(error) => return Err(RequestError::transport(&error)), + }; + + if stream { + let status = response.status(); + if !status.is_success() { + let content = response.text().await.unwrap_or_default(); + return Err(RequestError::new( + format!("http_{}", status.as_u16()), + truncate(&content, 500), + )); + } + let content_type = response + .headers() + .get(CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .unwrap_or("") + .to_string(); + if !content_type.contains("text/event-stream") { + let content = response.text().await.unwrap_or_default(); + return Err(RequestError::new( + "invalid_stream", + format!( + "expected text/event-stream, received {content_type:?}: {}", + truncate(&content, 300) + ), + )); + } + let mut bytes = response.bytes_stream(); + let mut pending = Vec::new(); + let mut validator = StreamValidator::new(endpoint); + while let Some(chunk) = bytes.next().await { + match chunk { + Ok(chunk) => { + pending.extend_from_slice(&chunk); + while let Some(newline) = pending.iter().position(|byte| *byte == b'\n') { + let mut line = pending.drain(..=newline).collect::>(); + line.pop(); + validator.read_line(&line)?; + } + } + Err(error) => { + return Err(RequestError::transport(&error)); + } + } + } + if !pending.is_empty() { + validator.read_line(&pending)?; + } + return validator.finish(); + } + + let status = response.status(); + if !status.is_success() { + let content = response.text().await.unwrap_or_default(); + return Err(RequestError::new( + format!("http_{}", status.as_u16()), + truncate(&content, 500), + )); + } + let text = match response.text().await { + Ok(text) => text, + Err(error) => return Err(RequestError::transport(&error)), + }; + let payload: Value = match serde_json::from_str(&text) { + Ok(payload) => payload, + Err(error) => return Err(RequestError::new("invalid_json", error.to_string())), + }; + if !payload.is_object() || payload.get("error").is_some() { + return Err(RequestError::new("invalid_response", truncate(&text, 500))); + } + let field = endpoint.required_field(); + if payload.get(field).is_none() { + return Err(RequestError::new( + "invalid_response", + format!( + "successful {} response did not contain {field:?}: {}", + endpoint.as_str(), + truncate(&text, 300) + ), + )); + } + Ok(()) +} + +/// Check liveness and verify that the requested model is advertised. +pub async fn preflight(client: &Client, base_url: &str, model: &str) -> Result<(), String> { + let health = client + .get(format!("{base_url}/health")) + .send() + .await + .map_err(|error| error.to_string())? + .error_for_status() + .map_err(|error| error.to_string())?; + let health_text = health.text().await.map_err(|error| error.to_string())?; + let health_body: Value = serde_json::from_str(&health_text).map_err(|_| { + format!( + "GET /health did not return JSON: {}", + truncate(&health_text, 300) + ) + })?; + if health_body.get("status").and_then(Value::as_str) != Some("ok") { + return Err(format!( + "GET /health returned an unexpected body: {}", + truncate(&health_text, 300) + )); + } + + let response = client + .get(format!("{base_url}/v1/models")) + .send() + .await + .map_err(|error| error.to_string())? + .error_for_status() + .map_err(|error| error.to_string())?; + let text = response.text().await.map_err(|error| error.to_string())?; + let body: Value = serde_json::from_str(&text).map_err(|_| { + format!( + "GET /v1/models did not return JSON: {}", + truncate(&text, 300) + ) + })?; + let entries = body.get("data").and_then(Value::as_array).ok_or_else(|| { + format!( + "GET /v1/models returned an unexpected body: {}", + truncate(&text, 300) + ) + })?; + if !entries + .iter() + .any(|entry| entry.get("id").and_then(Value::as_str) == Some(model)) + { + return Err(format!("model {model:?} is not listed by GET /v1/models")); + } + Ok(()) +} + +pub struct ServerState { + pub healthy: bool, + pub requests: Option, + pub errors: Option, +} + +/// Read liveness and cumulative metrics; one bad read becomes one failed sample. +pub async fn read_server_state(client: &Client, base_url: &str) -> ServerState { + let healthy = match client.get(format!("{base_url}/health")).send().await { + Ok(response) if response.status() == reqwest::StatusCode::OK => response + .text() + .await + .ok() + .and_then(|text| serde_json::from_str::(&text).ok()) + .and_then(|body| { + body.get("status") + .and_then(Value::as_str) + .map(|s| s == "ok") + }) + .unwrap_or(false), + _ => false, + }; + let metrics = match client.get(format!("{base_url}/metrics")).send().await { + Ok(response) if response.status().is_success() => response + .text() + .await + .map(|text| parse_metrics(&text)) + .unwrap_or_default(), + _ => Metrics::default(), + }; + ServerState { + healthy, + requests: metrics.requests, + errors: metrics.errors, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn request_bodies_match_public_endpoints() { + let chat = request_body(Endpoint::Chat, "route", "hello", 8, true); + let messages = request_body(Endpoint::Messages, "route", "hello", 8, true); + let responses = request_body(Endpoint::Responses, "route", "hello", 8, true); + + assert_eq!(chat["messages"][0]["content"], json!("hello")); + assert_eq!(chat["max_tokens"], json!(8)); + assert_eq!(messages["max_tokens"], json!(8)); + assert_eq!(responses["input"], json!("hello")); + assert_eq!(responses["max_output_tokens"], json!(8)); + for body in [&chat, &messages, &responses] { + assert_eq!(body["stream"], json!(true)); + } + } + + #[test] + fn parse_metrics_reads_only_required_counters() { + let metrics = parse_metrics( + "# TYPE switchyard_total_requests gauge\n\ + switchyard_total_requests 42\n\ + switchyard_total_errors{} 3\n\ + switchyard_requests_total{model=\"route\"} 10\n", + ); + + assert_eq!(metrics.requests, Some(42.0)); + assert_eq!(metrics.errors, Some(3.0)); + } + + #[test] + fn stream_validator_accepts_each_public_terminal_event() -> Result<(), RequestError> { + for (endpoint, stream) in [ + (Endpoint::Chat, "data: {\"choices\":[]}\n\ndata: [DONE]\n\n"), + ( + Endpoint::Messages, + "event: message_stop\ndata: {\"type\":\"message_stop\"}\n\n", + ), + ( + Endpoint::Responses, + "event: response.completed\ndata: {\"type\":\"response.completed\"}\n\n", + ), + ] { + let mut validator = StreamValidator::new(endpoint); + for line in stream.split('\n') { + validator.read_line(line.as_bytes())?; + } + validator.finish()?; + } + Ok(()) + } + + #[test] + fn stream_validator_rejects_errors_and_missing_terminal_events() { + let mut error = StreamValidator::new(Endpoint::Chat); + let result = error.read_line(b"data: {\"error\":{\"message\":\"boom\"}}"); + assert_eq!(result.unwrap_err().kind, "stream_error"); + + let mut incomplete = StreamValidator::new(Endpoint::Responses); + incomplete + .read_line(b"event: response.output_text.delta") + .expect("event name should parse"); + incomplete + .read_line(b"data: {\"type\":\"response.output_text.delta\",\"delta\":\"OK\"}") + .expect("data should parse"); + assert_eq!(incomplete.finish().unwrap_err().kind, "invalid_stream"); + } +} diff --git a/crates/switchyard-soak/src/lib.rs b/crates/switchyard-soak/src/lib.rs new file mode 100644 index 000000000..779b70093 --- /dev/null +++ b/crates/switchyard-soak/src/lib.rs @@ -0,0 +1,565 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#![doc = include_str!("../README.md")] + +mod client; +mod report; +mod scenarios; +mod stats; + +use std::fs; +use std::future::Future; +use std::path::{Path, PathBuf}; +use std::process::ExitCode; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::{Duration, Instant}; + +use clap::Parser; +use parking_lot::Mutex; +use reqwest::Client; +use reqwest::header::{AUTHORIZATION, HeaderMap, HeaderValue}; +use serde_json::json; +use tokio::sync::Notify; + +use crate::client::{Endpoint, preflight, send_request}; +use crate::report::{ResultsWriter, invalid_request_canary, reporter}; +use crate::scenarios::{Scenario, ScenarioOptions, ScenarioSet}; +use crate::stats::{RunStats, build_summary, now_utc_string, round3, utc_dir_stamp}; + +/// Command-line arguments for the soak test. +#[derive(Parser)] +#[command( + name = "switchyard-soak", + about = "Run a sustained, closed-loop load test against a live Switchyard server", + after_long_help = "Examples:\n switchyard-soak --model switchyard/general --duration 5m --concurrency 4\n switchyard-soak --model switchyard/general --scenario tool-call-burst --duration 5m\n switchyard-soak --model switchyard/general --export-scenarios scenarios\n\nThis command does not require oha or AIPerf. scripts/benchmark_routing_algorithms.py uses the exported scenario files with both tools.", + version +)] +pub struct Args { + /// HTTP base URL of the Switchyard server under test. + #[arg(long, default_value = "http://127.0.0.1:4000")] + base_url: String, + + /// Exact route id advertised by GET /v1/models. + #[arg(long)] + model: String, + + /// Time to keep sending load; use an s, m, or h suffix. + #[arg(long, value_parser = stats::parse_duration, default_value = "48h")] + duration: f64, + + /// Requests kept in flight; each worker sends its next request after the last one ends. + #[arg(long, default_value_t = 16)] + concurrency: usize, + + /// Default output-token limit; decode-heavy uses bounded 512 and 1,024 limits. + #[arg(long, default_value_t = 32)] + max_output_tokens: u32, + + /// Bytes of repeated prefix added to each prompt to exercise request memory and caching. + #[arg(long, default_value_t = 1024)] + prompt_bytes: usize, + + /// Named scenario collection; explicit --scenario values take precedence. + #[arg(long, value_enum, default_value_t = ScenarioSet::Standard)] + scenario_set: ScenarioSet, + + /// Exact scenario id to run; repeat to select more than one scenario. + #[arg(long, action = clap::ArgAction::Append)] + scenario: Vec, + + /// Context-window size used to bound generated long-context scenarios. + #[arg(long, default_value_t = 32_768)] + context_window_tokens: usize, + + /// Export selected AIPerf input files and exit without contacting a server. + #[arg(long)] + export_scenarios: Option, + + /// Seconds allowed to connect or wait between response bytes; an active stream may run longer. + #[arg(long, default_value_t = 120.0)] + request_timeout: f64, + + /// Seconds between health, metrics, process, and result samples. + #[arg(long, default_value_t = 60.0)] + report_interval: f64, + + /// Seconds between malformed-request checks; 0 turns this check off. + #[arg(long, default_value_t = 300.0)] + invalid_canary_interval: f64, + + /// Largest passing request-error fraction, from 0 (none) through 1 (all). + #[arg(long, default_value_t = 0.0)] + max_error_rate: f64, + + /// PID of a local switchyard-server process whose RSS and CPU should be sampled. + #[arg(long)] + server_pid: Option, + + /// Largest passing first-to-last RSS increase in MiB; requires --server-pid. + #[arg(long, requires = "server_pid")] + max_rss_growth_mib: Option, + + /// Name of an environment variable that holds the endpoint's bearer token. + #[arg(long)] + api_key_env: Option, + + /// New directory to create for config, interval, error, and summary files. + #[arg(long)] + results_dir: Option, +} + +impl Args { + /// Reject invalid numeric combinations after clap parses their types. + pub fn validate(&self) -> Result<(), String> { + if self.concurrency == 0 { + return Err("--concurrency must be greater than zero".to_string()); + } + if self.max_output_tokens == 0 || self.prompt_bytes == 0 { + return Err( + "--max-output-tokens and --prompt-bytes must be greater than zero".to_string(), + ); + } + ScenarioOptions { + model: &self.model, + prompt_bytes: self.prompt_bytes, + max_output_tokens: self.max_output_tokens, + context_window_tokens: self.context_window_tokens, + } + .validate()?; + if !self.request_timeout.is_finite() + || !self.report_interval.is_finite() + || self.request_timeout <= 0.0 + || self.report_interval <= 0.0 + { + return Err( + "--request-timeout and --report-interval must be greater than zero".to_string(), + ); + } + if !self.invalid_canary_interval.is_finite() || self.invalid_canary_interval < 0.0 { + return Err("--invalid-canary-interval must be zero or greater".to_string()); + } + if !(0.0..=1.0).contains(&self.max_error_rate) { + return Err("--max-error-rate must be between 0 and 1".to_string()); + } + if self.server_pid == Some(0) { + return Err("--server-pid must be greater than zero".to_string()); + } + if self + .max_rss_growth_mib + .is_some_and(|growth| !growth.is_finite() || growth < 0.0) + { + return Err("--max-rss-growth-mib must be zero or greater".to_string()); + } + Ok(()) + } +} + +#[derive(Clone)] +struct RunContext { + client: Client, + base_url: String, + stop: Arc, + stats: Arc>, + writer: Arc>, +} + +struct Workload { + model: String, + scenarios: Vec, +} + +/// A one-shot stop signal that many tasks can wait on and any task can raise. +struct Stop { + flag: AtomicBool, + notify: Notify, +} + +impl Stop { + fn new() -> Self { + Self { + flag: AtomicBool::new(false), + notify: Notify::new(), + } + } + + fn set(&self) { + self.flag.store(true, Ordering::SeqCst); + self.notify.notify_waiters(); + } + + fn is_set(&self) -> bool { + self.flag.load(Ordering::SeqCst) + } + + /// Resolve once the signal is raised, now or later. + async fn wait(&self) { + loop { + if self.is_set() { + return; + } + // Register before the second flag check so a set() between the two still wakes us. + let notified = self.notify.notified(); + tokio::pin!(notified); + notified.as_mut().enable(); + if self.is_set() { + return; + } + notified.await; + } + } +} + +/// Send closed-loop traffic until the run stops. +async fn worker( + context: RunContext, + workload: Arc, + worker_id: usize, +) -> Result<(), String> { + let mut request_number = 0; + while !context.stop.is_set() { + let scenario = &workload.scenarios[request_number % workload.scenarios.len()]; + let stream = (request_number / workload.scenarios.len()).is_multiple_of(2); + let request = scenario.request(request_number / workload.scenarios.len(), stream); + let session_id = format!("{}-worker-{worker_id}", request.session_id); + let started = Instant::now(); + let result = send_request( + &context.client, + &context.base_url, + request.endpoint, + &session_id, + request.body, + ) + .await; + let latency_ms = started.elapsed().as_secs_f64() * 1000.0; + context.stats.lock().record( + request.endpoint.as_str(), + scenario.id, + latency_ms, + result.as_ref().err().map(|error| error.kind.as_str()), + ); + if let Err(error) = result { + context + .writer + .lock() + .write_error(&json!({ + "timestamp_utc": now_utc_string(), + "worker": worker_id, + "scenario": scenario.id, + "endpoint": request.endpoint.as_str(), + "stream": stream, + "latency_ms": round3(latency_ms), + "error": error.kind, + "detail": error.detail, + })) + .map_err(|error| error.to_string())?; + } + request_number = request_number.wrapping_add(1); + } + Ok(()) +} + +/// Stop the run when a background task fails or panics. +async fn guard_stop( + stop: Arc, + task: impl Future>, +) -> Result<(), String> { + struct StopOnDrop(Arc, bool); + impl Drop for StopOnDrop { + fn drop(&mut self) { + if self.1 { + self.0.set(); + } + } + } + let mut guard = StopOnDrop(stop, true); + let result = task.await; + guard.1 = result.is_err(); + result +} + +/// Add one joined task's error to the failure reasons; cancelled tasks are expected. +fn collect_failure( + name: &str, + result: Result, tokio::task::JoinError>, + failures: &mut Vec, +) { + match result { + Ok(Ok(())) => {} + Ok(Err(reason)) => failures.push(format!("{name} failed: {reason}")), + Err(join) if join.is_cancelled() => {} + Err(join) => failures.push(format!("{name} failed: {join}")), + } +} + +/// Raise *stop* on SIGINT or SIGTERM so an operator can end the run cleanly. +fn spawn_signal_listener(stop: Arc) -> tokio::task::JoinHandle<()> { + tokio::spawn(async move { + #[cfg(unix)] + { + use tokio::signal::unix::{SignalKind, signal}; + let mut interrupt = signal(SignalKind::interrupt()).ok(); + let mut terminate = signal(SignalKind::terminate()).ok(); + let wait_interrupt = async { + match interrupt.as_mut() { + Some(stream) => { + stream.recv().await; + } + None => std::future::pending::<()>().await, + } + }; + let wait_terminate = async { + match terminate.as_mut() { + Some(stream) => { + stream.recv().await; + } + None => std::future::pending::<()>().await, + } + }; + tokio::select! { + _ = wait_interrupt => {} + _ = wait_terminate => {} + } + stop.set(); + } + #[cfg(not(unix))] + { + let _ = tokio::signal::ctrl_c().await; + stop.set(); + } + }) +} + +/// Record every non-secret input with the normalized duration and fixed request variants. +fn write_config( + results_dir: &Path, + args: &Args, + model: &str, + scenarios: &[Scenario], +) -> Result<(), String> { + let config = json!({ + "base_url": args.base_url, + "model": model, + "duration_seconds": args.duration, + "concurrency": args.concurrency, + "endpoints": Endpoint::ALL.map(Endpoint::as_str), + "streaming": [true, false], + "max_output_tokens": args.max_output_tokens, + "prompt_bytes": args.prompt_bytes, + "context_window_tokens": args.context_window_tokens, + "scenarios": scenarios.iter().map(|scenario| scenario.id).collect::>(), + "request_timeout": args.request_timeout, + "report_interval": args.report_interval, + "invalid_canary_interval": args.invalid_canary_interval, + "max_error_rate": args.max_error_rate, + "server_pid": args.server_pid, + "max_rss_growth_mib": args.max_rss_growth_mib, + "api_key_env": args.api_key_env, + }); + let body = serde_json::to_string_pretty(&config).map_err(|error| error.to_string())?; + fs::write(results_dir.join("config.json"), format!("{body}\n")) + .map_err(|error| error.to_string()) +} + +/// Run the configured soak test and return a process exit code (0 pass, 1 fail). +pub async fn run(args: Args) -> Result { + args.validate()?; + + let scenarios = scenarios::select( + ScenarioOptions { + model: &args.model, + prompt_bytes: args.prompt_bytes, + max_output_tokens: args.max_output_tokens, + context_window_tokens: args.context_window_tokens, + }, + args.scenario_set, + &args.scenario, + )?; + if let Some(output_dir) = &args.export_scenarios { + let manifest = scenarios::export(&scenarios, &args.model, output_dir)?; + println!("Exported scenario manifest: {}", manifest.display()); + return Ok(0); + } + + let token = match &args.api_key_env { + Some(var) => { + let token = std::env::var(var).ok().filter(|value| !value.is_empty()); + if token.is_none() { + return Err(format!("${var} is not set")); + } + token + } + None => None, + }; + + // Per-operation timeouts, not a whole-request deadline: a healthy long stream that keeps + // delivering bytes must not be aborted, so bound connect time and idle time between reads. + let request_timeout = Duration::from_secs_f64(args.request_timeout); + let mut builder = Client::builder() + .no_proxy() + .connect_timeout(request_timeout) + .read_timeout(request_timeout) + .pool_max_idle_per_host(args.concurrency + 4); + if let Some(token) = &token { + let mut headers = HeaderMap::new(); + let value = HeaderValue::from_str(&format!("Bearer {token}")) + .map_err(|_| "the bearer token is not a valid HTTP header value".to_string())?; + headers.insert(AUTHORIZATION, value); + builder = builder.default_headers(headers); + } + let client = builder.build().map_err(|error| error.to_string())?; + let base_url = args.base_url.trim_end_matches('/').to_string(); + + preflight(&client, &base_url, &args.model).await?; + let results_dir = args + .results_dir + .clone() + .unwrap_or_else(|| PathBuf::from("soak-results").join(utc_dir_stamp())); + let writer = Arc::new(Mutex::new( + ResultsWriter::new(&results_dir).map_err(|error| error.to_string())?, + )); + write_config(&results_dir, &args, &args.model, &scenarios)?; + + println!( + "Soak started: model={} duration={}s concurrency={} scenarios={} results={}", + args.model, + args.duration, + args.concurrency, + scenarios + .iter() + .map(|scenario| scenario.id) + .collect::>() + .join(","), + results_dir.display(), + ); + + let started = Instant::now(); + let stats = Arc::new(Mutex::new(RunStats::new(2026))); + let stop = Arc::new(Stop::new()); + let workers_done = Arc::new(Stop::new()); + let context = RunContext { + client, + base_url, + stop: stop.clone(), + stats: stats.clone(), + writer: writer.clone(), + }; + + let signal_listener = spawn_signal_listener(stop.clone()); + + let deadline = { + let stop = stop.clone(); + let stats = stats.clone(); + let duration = Duration::from_secs_f64(args.duration); + tokio::spawn(async move { + tokio::time::sleep(duration).await; + stats.lock().completed_duration = true; + stop.set(); + }) + }; + + let workload = Arc::new(Workload { + model: args.model.clone(), + scenarios, + }); + let mut worker_handles = Vec::new(); + for worker_id in 0..args.concurrency { + let handle = tokio::spawn(guard_stop( + stop.clone(), + worker(context.clone(), workload.clone(), worker_id), + )); + worker_handles.push((format!("worker-{worker_id}"), handle)); + } + let reporter_handle = tokio::spawn(guard_stop( + stop.clone(), + reporter( + context.clone(), + started, + Duration::from_secs_f64(args.report_interval), + args.duration, + args.server_pid, + workers_done.clone(), + ), + )); + let canary_handle = tokio::spawn(guard_stop( + stop.clone(), + invalid_request_canary(context, args.invalid_canary_interval, workload), + )); + + stop.wait().await; + deadline.abort(); + signal_listener.abort(); + + // A crashed worker/reporter/canary must not discard the run: record it as a failure reason + // so the summary is still written and the run fails closed. + let mut task_failures = Vec::new(); + for (name, handle) in worker_handles { + collect_failure(&name, handle.await, &mut task_failures); + } + workers_done.set(); + collect_failure("reporter", reporter_handle.await, &mut task_failures); + collect_failure( + "invalid-request-canary", + canary_handle.await, + &mut task_failures, + ); + + let elapsed = started.elapsed().as_secs_f64(); + let (error_records, dropped_error_records) = { writer.lock().error_counts() }; + let summary = build_summary( + &stats.lock(), + elapsed, + args.max_error_rate, + args.max_rss_growth_mib, + error_records, + dropped_error_records, + &task_failures, + ); + let summary_path = results_dir.join("summary.json"); + let summary_body = serde_json::to_string_pretty(&summary).map_err(|error| error.to_string())?; + fs::write(&summary_path, format!("{summary_body}\n")).map_err(|error| error.to_string())?; + + let label = if summary.passed { "PASS" } else { "FAIL" }; + println!( + "Soak {label}: requests={} error_rate={:.4}% p95_ms={} summary={}", + summary.requests, + summary.error_rate * 100.0, + summary + .latency_p95_ms + .map(|value| value.to_string()) + .unwrap_or_default(), + summary_path.display(), + ); + for reason in &summary.failure_reasons { + println!("- {reason}"); + } + Ok(if summary.passed { 0 } else { 1 }) +} + +/// Parse arguments, run the test on a multi-thread runtime, and map the result to an exit code. +pub fn cli_main() -> ExitCode { + let args = Args::parse(); + if let Err(message) = args.validate() { + eprintln!("switchyard-soak: {message}"); + return ExitCode::from(2); + } + let runtime = match tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + { + Ok(runtime) => runtime, + Err(error) => { + eprintln!("soak test setup failed: {error}"); + return ExitCode::from(2); + } + }; + match runtime.block_on(run(args)) { + Ok(0) => ExitCode::SUCCESS, + Ok(_) => ExitCode::from(1), + Err(error) => { + eprintln!("soak test setup failed: {error}"); + ExitCode::from(2) + } + } +} diff --git a/crates/switchyard-soak/src/main.rs b/crates/switchyard-soak/src/main.rs new file mode 100644 index 000000000..8e3d676d5 --- /dev/null +++ b/crates/switchyard-soak/src/main.rs @@ -0,0 +1,10 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Binary entrypoint for `switchyard-soak`. + +use std::process::ExitCode; + +fn main() -> ExitCode { + switchyard_soak::cli_main() +} diff --git a/crates/switchyard-soak/src/report.rs b/crates/switchyard-soak/src/report.rs new file mode 100644 index 000000000..69e512528 --- /dev/null +++ b/crates/switchyard-soak/src/report.rs @@ -0,0 +1,327 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Result files plus the background reporter and invalid-request canary tasks. + +use std::fs::{self, File}; +use std::io::{self, Write}; +use std::path::Path; +use std::time::{Duration, Instant}; + +use serde_json::{Value, json}; + +use crate::client::read_server_state; +use crate::stats::{latency_stats, now_utc_string, round3}; +use crate::{RunContext, Stop, Workload}; + +/// Cap on individually recorded failures so a bad run cannot fill the disk. +const MAX_ERROR_RECORDS: u64 = 10_000; + +const INTERVAL_FIELDS: [&str; 15] = [ + "timestamp_utc", + "elapsed_seconds", + "requests", + "successes", + "failures", + "requests_per_second", + "latency_p50_ms", + "latency_p95_ms", + "latency_p99_ms", + "latency_max_ms", + "health", + "server_total_requests", + "server_total_errors", + "rss_mib", + "cpu_percent", +]; + +/// Write interval rows and bounded error details as the run proceeds. +pub struct ResultsWriter { + interval_file: File, + error_file: File, + error_records: u64, + dropped_error_records: u64, +} + +impl ResultsWriter { + /// Create a fresh results directory; fails if it already exists. + pub fn new(results_dir: &Path) -> io::Result { + if let Some(parent) = results_dir.parent() + && !parent.as_os_str().is_empty() + { + fs::create_dir_all(parent)?; + } + fs::create_dir(results_dir)?; + let mut interval_file = File::create(results_dir.join("intervals.csv"))?; + writeln!(interval_file, "{}", INTERVAL_FIELDS.join(","))?; + let error_file = File::create(results_dir.join("errors.jsonl"))?; + Ok(Self { + interval_file, + error_file, + error_records: 0, + dropped_error_records: 0, + }) + } + + /// Append one interval row; `cells` are already formatted in `INTERVAL_FIELDS` order. + pub fn write_interval(&mut self, cells: &[String]) -> io::Result<()> { + writeln!(self.interval_file, "{}", cells.join(","))?; + self.interval_file.flush() + } + + /// Append one failure record, dropping past the bound instead of growing without limit. + pub fn write_error(&mut self, record: &Value) -> io::Result<()> { + if self.error_records >= MAX_ERROR_RECORDS { + self.dropped_error_records += 1; + return Ok(()); + } + writeln!(self.error_file, "{record}")?; + self.error_file.flush()?; + self.error_records += 1; + Ok(()) + } + + pub fn error_counts(&self) -> (u64, u64) { + (self.error_records, self.dropped_error_records) + } +} + +/// Format an optional number for a CSV cell; `None` becomes an empty cell. +fn cell(value: Option) -> String { + value + .map(|value| round3(value).to_string()) + .unwrap_or_default() +} + +/// Return RSS MiB and CPU percent for *pid* using the local `ps` command. +pub async fn process_sample(pid: Option) -> (Option, Option) { + let Some(pid) = pid else { + return (None, None); + }; + let output = match tokio::process::Command::new("ps") + .args(["-o", "rss=,pcpu=", "-p", &pid.to_string()]) + .output() + .await + { + Ok(output) => output, + // ps missing or fork failed: record a process-check miss, don't crash the reporter. + Err(_) => return (None, None), + }; + if !output.status.success() { + return (None, None); + } + let stdout = String::from_utf8_lossy(&output.stdout); + let mut fields = stdout.split_whitespace(); + let (Some(rss), Some(cpu), None) = (fields.next(), fields.next(), fields.next()) else { + return (None, None); + }; + match (rss.parse::(), cpu.parse::()) { + (Ok(rss_kib), Ok(cpu_percent)) => (Some(rss_kib / 1024.0), Some(cpu_percent)), + _ => (None, None), + } +} + +/// Write one liveness, metrics, resource, and latency row per interval. +pub async fn reporter( + context: RunContext, + started: Instant, + interval: Duration, + target_seconds: f64, + server_pid: Option, + workers_done: std::sync::Arc, +) -> Result<(), String> { + let mut previous_report = started; + loop { + tokio::select! { + _ = tokio::time::sleep(interval) => {} + _ = context.stop.wait() => {} + } + // Once stopping, wait for the workers to drain so the final row counts their last requests. + if context.stop.is_set() { + workers_done.wait().await; + } + + let now = Instant::now(); + let mut interval_stats = context.stats.lock().take_interval(); + let server = read_server_state(&context.client, &context.base_url).await; + let (rss_mib, cpu_percent) = process_sample(server_pid).await; + + let (total_successes, total_failures) = { + let mut state = context.stats.lock(); + state.health_checks += 1; + state.metrics_checks += 1; + if !server.healthy { + state.health_failures += 1; + } + if server.requests.is_none() || server.errors.is_none() { + state.metrics_failures += 1; + } + if server_pid.is_some() { + state.process_checks += 1; + if rss_mib.is_none() || cpu_percent.is_none() { + state.process_failures += 1; + } + } + if let Some(rss) = rss_mib { + state.rss_samples.push(rss); + } + if let (Some(current), Some(previous)) = + (server.requests, state.previous_server_requests) + && current < previous + { + state.server_restarts += 1; + } + if let Some(current) = server.requests { + state.previous_server_requests = Some(current); + } + (state.total_successes, state.total_failures) + }; + + let elapsed_interval = (now - previous_report).as_secs_f64().max(0.001); + let requests = interval_stats.successes + interval_stats.failures; + let latency = latency_stats(&mut interval_stats.latencies_ms); + let requests_per_second = round3(requests as f64 / elapsed_interval); + let timestamp = now_utc_string(); + let elapsed_seconds = round3((now - started).as_secs_f64()); + let health_label = if server.healthy { "ok" } else { "failed" }; + + let cells = vec![ + timestamp.clone(), + elapsed_seconds.to_string(), + requests.to_string(), + interval_stats.successes.to_string(), + interval_stats.failures.to_string(), + requests_per_second.to_string(), + cell(latency.p50_ms), + cell(latency.p95_ms), + cell(latency.p99_ms), + cell(latency.max_ms), + health_label.to_string(), + cell(server.requests), + cell(server.errors), + cell(rss_mib), + cell(cpu_percent), + ]; + // Cumulative, progress, and a glanceable status token so a remote tail of the log shows + // at once that the run is alive, how far along it is, and whether it is healthy. + let cumulative_requests = total_successes + total_failures; + let cumulative_error_rate = if cumulative_requests > 0 { + total_failures as f64 / cumulative_requests as f64 + } else { + 0.0 + }; + let progress = (elapsed_seconds / target_seconds).min(1.0); + let status = if requests == 0 { + "stalled" + } else if !server.healthy || interval_stats.failures > 0 { + "degraded" + } else { + "ok" + }; + let p95_text = latency.p95_ms.map(|v| v.to_string()).unwrap_or_default(); + let rss_text = rss_mib.map(|v| round3(v).to_string()).unwrap_or_default(); + + context + .writer + .lock() + .write_interval(&cells) + .map_err(|error| error.to_string())?; + + println!( + "[{timestamp}] progress={elapsed_seconds:.0}s/{target_seconds:.0}s({:.0}%) \ + reqs={cumulative_requests} interval={requests} errors={total_failures}({:.4}%) \ + rps={requests_per_second} p95_ms={p95_text} health={health_label} rss_mib={rss_text} \ + status={}", + progress * 100.0, + cumulative_error_rate * 100.0, + status.to_uppercase(), + ); + + previous_report = now; + if context.stop.is_set() { + return Ok(()); + } + } +} + +/// Confirm invalid input returns 400 and the server stays live, on a fixed interval. +pub async fn invalid_request_canary( + context: RunContext, + interval: f64, + workload: std::sync::Arc, +) -> Result<(), String> { + if interval <= 0.0 { + return Ok(()); + } + let interval = Duration::from_secs_f64(interval); + loop { + tokio::select! { + _ = tokio::time::sleep(interval) => {} + _ = context.stop.wait() => {} + } + if context.stop.is_set() { + return Ok(()); + } + + context.stats.lock().canaries += 1; + let probe = async { + let invalid = context + .client + .post(format!("{}/v1/chat/completions", context.base_url)) + .json(&json!({"model": workload.model, "messages": "invalid"})) + .send() + .await?; + let health = context + .client + .get(format!("{}/health", context.base_url)) + .send() + .await?; + Ok::<(u16, u16), reqwest::Error>((invalid.status().as_u16(), health.status().as_u16())) + } + .await; + let (passed, detail) = match probe { + Ok((invalid_status, health_status)) => ( + invalid_status == 400 && health_status == 200, + format!("invalid_status={invalid_status}, health_status={health_status}"), + ), + Err(error) => (false, error.to_string()), + }; + if !passed { + context.stats.lock().canary_failures += 1; + context + .writer + .lock() + .write_error(&json!({ + "timestamp_utc": now_utc_string(), + "error": "invalid_request_canary", + "detail": detail, + })) + .map_err(|error| error.to_string())?; + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn error_details_stop_growing_at_the_file_limit() -> io::Result<()> { + let parent = tempfile::tempdir()?; + let results_dir = parent.path().join("results"); + let mut writer = ResultsWriter::new(&results_dir)?; + + for _ in 0..=MAX_ERROR_RECORDS { + writer.write_error(&serde_json::json!({"error": "upstream"}))?; + } + + assert_eq!(writer.error_counts(), (MAX_ERROR_RECORDS, 1)); + assert_eq!( + fs::read_to_string(results_dir.join("errors.jsonl"))? + .lines() + .count(), + 10_000 + ); + Ok(()) + } +} diff --git a/crates/switchyard-soak/src/scenarios/classifier_mix.rs b/crates/switchyard-soak/src/scenarios/classifier_mix.rs new file mode 100644 index 000000000..893366fdf --- /dev/null +++ b/crates/switchyard-soak/src/scenarios/classifier_mix.rs @@ -0,0 +1,42 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use super::{ + ErrorExpectation, Scenario, ScenarioGroup, ScenarioOptions, Session, chat_payload, user, +}; + +pub fn build(options: ScenarioOptions<'_>) -> Scenario { + let sessions = (0..20) + .map(|index| { + let hard = (8..10).contains(&index) || index >= 15; + let marker = if hard { + "classifier_hard" + } else { + "classifier_easy" + }; + Session { + session_id: format!("classifier-mix-{index}"), + payloads: vec![chat_payload( + options.model, + vec![user(format!( + "[scenario:{marker}] Request {index}: {}", + if hard { + "reason about a distributed failure with incomplete evidence" + } else { + "return the sum of two and two" + } + ))], + options.max_output_tokens, + )], + } + }) + .collect(); + Scenario::chat( + "classifier-mix", + ScenarioGroup::Agentic, + "Two deterministic easy/hard mixes: 80/20 followed by 50/50.", + "Selection shares match the input mix and classifier overhead is visible.", + ErrorExpectation::SUCCESS, + sessions, + ) +} diff --git a/crates/switchyard-soak/src/scenarios/client_cancellation.rs b/crates/switchyard-soak/src/scenarios/client_cancellation.rs new file mode 100644 index 000000000..66ccf29e3 --- /dev/null +++ b/crates/switchyard-soak/src/scenarios/client_cancellation.rs @@ -0,0 +1,26 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use super::{ + ErrorExpectation, Scenario, ScenarioGroup, ScenarioOptions, Session, chat_payload, user, +}; + +pub fn build(options: ScenarioOptions<'_>) -> Scenario { + Scenario::chat( + "client-cancellation", + ScenarioGroup::Resilience, + "Slow streaming requests whose client timeout cancels work in flight.", + "Cancellation releases connections and does not destabilize later traffic.", + ErrorExpectation::ALL, + vec![Session { + session_id: "client-cancellation".to_string(), + payloads: vec![chat_payload( + options.model, + vec![user( + "[scenario:client_cancellation] Delay this response until the client leaves.", + )], + options.max_output_tokens, + )], + }], + ) +} diff --git a/crates/switchyard-soak/src/scenarios/context_overflow.rs b/crates/switchyard-soak/src/scenarios/context_overflow.rs new file mode 100644 index 000000000..3e57e2007 --- /dev/null +++ b/crates/switchyard-soak/src/scenarios/context_overflow.rs @@ -0,0 +1,26 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use super::{ + ErrorExpectation, Scenario, ScenarioGroup, ScenarioOptions, Session, chat_payload, token_text, + user, +}; + +pub fn build(options: ScenarioOptions<'_>) -> Scenario { + let tokens = (options.context_window_tokens * 9 / 10).max(1); + Scenario::chat( + "context-overflow", + ScenarioGroup::Resilience, + "A near-window request that one configured target rejects as too long.", + "A multi-target route retries an eligible target and returns a valid response.", + ErrorExpectation::SUCCESS, + vec![Session { + session_id: "context-overflow".to_string(), + payloads: vec![chat_payload( + options.model, + vec![user(token_text(tokens, "context_overflow"))], + options.max_output_tokens, + )], + }], + ) +} diff --git a/crates/switchyard-soak/src/scenarios/decode_heavy.rs b/crates/switchyard-soak/src/scenarios/decode_heavy.rs new file mode 100644 index 000000000..b19c4d68a --- /dev/null +++ b/crates/switchyard-soak/src/scenarios/decode_heavy.rs @@ -0,0 +1,30 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use super::{ + ErrorExpectation, Scenario, ScenarioGroup, ScenarioOptions, Session, chat_payload, user, +}; + +pub fn build(options: ScenarioOptions<'_>) -> Scenario { + let sessions = [512, 1024] + .into_iter() + .map(|tokens| Session { + session_id: format!("decode-heavy-{tokens}"), + payloads: vec![chat_payload( + options.model, + vec![user(format!( + "[scenario:decode_heavy] Write {tokens} numbered one-word items." + ))], + tokens, + )], + }) + .collect(); + Scenario::chat( + "decode-heavy", + ScenarioGroup::Core, + "Short inputs with 512-token and 1,024-token output limits.", + "Output token throughput stays stable while long streams remain valid.", + ErrorExpectation::SUCCESS, + sessions, + ) +} diff --git a/crates/switchyard-soak/src/scenarios/failure_pressure.rs b/crates/switchyard-soak/src/scenarios/failure_pressure.rs new file mode 100644 index 000000000..a1606b065 --- /dev/null +++ b/crates/switchyard-soak/src/scenarios/failure_pressure.rs @@ -0,0 +1,36 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use super::{ + ErrorExpectation, Scenario, ScenarioGroup, ScenarioOptions, Session, chat_payload, user, +}; + +pub fn build(options: ScenarioOptions<'_>) -> Scenario { + let cases = [ + ("truncated-stream", "truncated_stream"), + ("upstream-429", "upstream_429"), + ("upstream-500", "upstream_500"), + ("classifier-invalid", "classifier_invalid"), + ]; + let sessions = cases + .into_iter() + .map(|(id, marker)| Session { + session_id: id.to_string(), + payloads: vec![chat_payload( + options.model, + vec![user(format!( + "[scenario:{marker}] Exercise the configured recovery path." + ))], + options.max_output_tokens, + )], + }) + .collect(); + Scenario::chat( + "failure-pressure", + ScenarioGroup::Resilience, + "Bounded 429, 500, malformed-classifier, and truncated-stream injections.", + "Retries recover transient failures; terminal failures remain explicit and bounded.", + ErrorExpectation::MIXED, + sessions, + ) +} diff --git a/crates/switchyard-soak/src/scenarios/growing_conversation.rs b/crates/switchyard-soak/src/scenarios/growing_conversation.rs new file mode 100644 index 000000000..17da931aa --- /dev/null +++ b/crates/switchyard-soak/src/scenarios/growing_conversation.rs @@ -0,0 +1,34 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use super::{ + ErrorExpectation, Scenario, ScenarioGroup, ScenarioOptions, Session, assistant, chat_payload, + user, +}; + +pub fn build(options: ScenarioOptions<'_>) -> Scenario { + let mut messages = Vec::new(); + let mut payloads = Vec::new(); + for turn in 0..8 { + messages.push(user(format!( + "[scenario:growing_conversation] Turn {turn}: remember the number {turn}." + ))); + payloads.push(chat_payload( + options.model, + messages.clone(), + options.max_output_tokens.min(64), + )); + messages.push(assistant(format!("I recorded {turn}."))); + } + Scenario::chat( + "growing-conversation", + ScenarioGroup::Agentic, + "An eight-turn session whose complete message history grows each turn.", + "Session affinity holds and latency tracks the growing history.", + ErrorExpectation::SUCCESS, + vec![Session { + session_id: "growing-conversation".to_string(), + payloads, + }], + ) +} diff --git a/crates/switchyard-soak/src/scenarios/large_tool_catalog.rs b/crates/switchyard-soak/src/scenarios/large_tool_catalog.rs new file mode 100644 index 000000000..a95363f18 --- /dev/null +++ b/crates/switchyard-soak/src/scenarios/large_tool_catalog.rs @@ -0,0 +1,61 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use serde_json::{Value, json}; + +use super::{ + ErrorExpectation, Scenario, ScenarioGroup, ScenarioOptions, Session, chat_payload, user, +}; + +fn tools(count: usize) -> Vec { + (0..count) + .map(|index| { + json!({ + "type": "function", + "function": { + "name": format!("lookup_record_{index}"), + "description": format!("Look up record {index} in the test catalog."), + "parameters": { + "type": "object", + "properties": { + "query": {"type": "string"}, + "limit": {"type": "integer", "minimum": 1, "maximum": 10} + }, + "required": ["query"], + "additionalProperties": false + } + } + }) + }) + .collect() +} + +pub fn build(options: ScenarioOptions<'_>) -> Scenario { + let sessions = [16, 64] + .into_iter() + .map(|count| { + let mut payload = chat_payload( + options.model, + vec![user(format!( + "[scenario:large_tool_catalog] Use lookup_record_{} for query green.", + count - 1 + ))], + options.max_output_tokens.min(128), + ); + payload["tools"] = Value::Array(tools(count)); + payload["tool_choice"] = json!("auto"); + Session { + session_id: format!("large-tool-catalog-{count}"), + payloads: vec![payload], + } + }) + .collect(); + Scenario::chat( + "large-tool-catalog", + ScenarioGroup::Agentic, + "Requests with 16-tool and 64-tool JSON-schema catalogs.", + "Routing overhead remains bounded while large tool schemas are forwarded intact.", + ErrorExpectation::SUCCESS, + sessions, + ) +} diff --git a/crates/switchyard-soak/src/scenarios/long_context.rs b/crates/switchyard-soak/src/scenarios/long_context.rs new file mode 100644 index 000000000..6958e847f --- /dev/null +++ b/crates/switchyard-soak/src/scenarios/long_context.rs @@ -0,0 +1,36 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use super::{ + ErrorExpectation, Scenario, ScenarioGroup, ScenarioOptions, Session, chat_payload, token_text, + user, +}; + +pub fn build(options: ScenarioOptions<'_>) -> Scenario { + let mut lengths = vec![8_192, 32_000, options.context_window_tokens * 9 / 10]; + lengths.retain(|length| *length < options.context_window_tokens); + lengths.sort_unstable(); + lengths.dedup(); + let sessions = lengths + .into_iter() + .map(|tokens| Session { + session_id: format!("long-context-{tokens}"), + payloads: vec![chat_payload( + options.model, + vec![user(format!( + "{}\nSummarize the final marker in one sentence.", + token_text(tokens, "long_context") + ))], + options.max_output_tokens.min(64), + )], + }) + .collect(); + Scenario::chat( + "long-context", + ScenarioGroup::Core, + "Requests at 8K, 32K, and near the configured context window.", + "TTFT rises with input length without routing errors or memory growth.", + ErrorExpectation::SUCCESS, + sessions, + ) +} diff --git a/crates/switchyard-soak/src/scenarios/mixed_traffic.rs b/crates/switchyard-soak/src/scenarios/mixed_traffic.rs new file mode 100644 index 000000000..ae1a444a4 --- /dev/null +++ b/crates/switchyard-soak/src/scenarios/mixed_traffic.rs @@ -0,0 +1,41 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use super::{ + ErrorExpectation, Scenario, ScenarioGroup, ScenarioOptions, Session, chat_payload, token_text, + user, +}; + +pub fn build(options: ScenarioOptions<'_>) -> Scenario { + let lengths = [16, 24, 32, 48, 64, 96, 128, 1_024, 2_048, 8_192]; + let sessions = lengths + .into_iter() + .enumerate() + .map(|(index, tokens)| Session { + session_id: format!("mixed-{index}"), + payloads: vec![chat_payload( + options.model, + vec![user(format!( + "{}\nReply with the request number {index}.", + token_text( + tokens.min(options.context_window_tokens / 2), + "mixed_traffic" + ) + ))], + if index == 9 { + 256 + } else { + options.max_output_tokens + }, + )], + }) + .collect(); + Scenario::chat( + "mixed-traffic", + ScenarioGroup::Core, + "A 70/20/10 mix of short, medium, and long requests.", + "Tail latency stays bounded when unlike request sizes share one route.", + ErrorExpectation::SUCCESS, + sessions, + ) +} diff --git a/crates/switchyard-soak/src/scenarios/mod.rs b/crates/switchyard-soak/src/scenarios/mod.rs new file mode 100644 index 000000000..b7bf9a2cf --- /dev/null +++ b/crates/switchyard-soak/src/scenarios/mod.rs @@ -0,0 +1,528 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Reproducible request scenarios shared by the soak command and AIPerf runner. + +mod classifier_mix; +mod client_cancellation; +mod context_overflow; +mod decode_heavy; +mod failure_pressure; +mod growing_conversation; +mod large_tool_catalog; +mod long_context; +mod mixed_traffic; +mod prefix_reuse; +mod short_interactive; +mod stage_transitions; +mod tool_call_burst; + +use std::collections::HashSet; +use std::fs; +use std::path::{Path, PathBuf}; + +use clap::ValueEnum; +use serde::Serialize; +use serde_json::{Value, json}; + +use crate::client::Endpoint; + +/// Largest accepted context-window input for generated scenario data. +const MAX_CONTEXT_WINDOW_TOKENS: usize = 1_000_000; +const MIN_CONTEXT_WINDOW_TOKENS: usize = 16_384; +const MAX_PROMPT_BYTES: usize = 1_000_000; + +/// Scenario family used by CLI selection and report grouping. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ScenarioGroup { + Core, + Agentic, + Resilience, +} + +/// Accepted client-visible error-rate range for one scenario. +#[derive(Clone, Copy, Serialize)] +pub struct ErrorExpectation { + pub min_rate: f64, + pub max_rate: f64, +} + +impl ErrorExpectation { + pub const SUCCESS: Self = Self { + min_rate: 0.0, + max_rate: 0.0, + }; + pub const MIXED: Self = Self { + min_rate: 0.01, + max_rate: 0.75, + }; + pub const ALL: Self = Self { + min_rate: 0.8, + max_rate: 1.0, + }; +} + +/// Named scenario collection selected by the soak and benchmark commands. +#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] +#[value(rename_all = "snake_case")] +pub enum ScenarioSet { + Core, + Agentic, + Resilience, + Standard, + All, +} + +impl ScenarioSet { + fn contains(self, group: ScenarioGroup) -> bool { + match self { + Self::Core => group == ScenarioGroup::Core, + Self::Agentic => group == ScenarioGroup::Agentic, + Self::Resilience => group == ScenarioGroup::Resilience, + Self::Standard => group != ScenarioGroup::Resilience, + Self::All => true, + } + } +} + +/// Request-generation limits shared by every scenario builder. +#[derive(Clone, Copy)] +pub struct ScenarioOptions<'a> { + pub model: &'a str, + pub prompt_bytes: usize, + pub max_output_tokens: u32, + pub context_window_tokens: usize, +} + +impl ScenarioOptions<'_> { + pub fn validate(&self) -> Result<(), String> { + if self.model.is_empty() { + return Err("scenario model must not be empty".to_string()); + } + if self.prompt_bytes == 0 || self.max_output_tokens == 0 { + return Err("scenario prompt and output limits must be greater than zero".to_string()); + } + if !(MIN_CONTEXT_WINDOW_TOKENS..=MAX_CONTEXT_WINDOW_TOKENS) + .contains(&self.context_window_tokens) + { + return Err(format!( + "--context-window-tokens must be between {MIN_CONTEXT_WINDOW_TOKENS} and {MAX_CONTEXT_WINDOW_TOKENS}" + )); + } + let prompt_limit = MAX_PROMPT_BYTES.min(self.context_window_tokens.saturating_mul(2)); + if self.prompt_bytes > prompt_limit { + return Err(format!( + "--prompt-bytes must not exceed {prompt_limit} for this context window" + )); + } + Ok(()) + } +} + +/// One AIPerf conversation; payloads remain ordered within the session. +#[derive(Clone, Serialize)] +pub struct Session { + pub session_id: String, + pub payloads: Vec, +} + +/// One request the Rust soak command can send. +pub struct RequestCase { + pub endpoint: Endpoint, + pub session_id: String, + streaming_body: Value, + nonstreaming_body: Value, +} + +impl RequestCase { + pub fn new(endpoint: Endpoint, session_id: String, streaming_body: Value) -> Self { + let mut nonstreaming_body = streaming_body.clone(); + nonstreaming_body["stream"] = Value::Bool(false); + Self { + endpoint, + session_id, + streaming_body, + nonstreaming_body, + } + } +} + +pub struct PreparedRequest<'a> { + pub endpoint: Endpoint, + pub session_id: &'a str, + pub body: &'a Value, +} + +/// QPS point expressed relative to the benchmark command's base request rate. +#[derive(Clone, Serialize)] +pub struct RatePoint { + pub time_s: u32, + pub rate_multiplier: f64, +} + +/// AIPerf schedule applied to a request scenario. +#[derive(Clone, Serialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum LoadProfile { + Fixed { + id: &'static str, + }, + ConcurrencyKnee { + id: &'static str, + concurrency_steps: Vec, + }, + TrafficBurst { + id: &'static str, + duration_seconds: u32, + points: Vec, + }, +} + +/// Immutable scenario data built once before load workers start. +pub struct Scenario { + pub id: &'static str, + pub group: ScenarioGroup, + pub description: &'static str, + pub expected: &'static str, + pub expected_error_rate: ErrorExpectation, + pub sessions: Vec, + pub soak_requests: Vec, + pub load_profiles: Vec, +} + +impl Scenario { + /// Build a Chat Completions scenario whose AIPerf payloads also drive soak traffic. + pub fn chat( + id: &'static str, + group: ScenarioGroup, + description: &'static str, + expected: &'static str, + expected_error_rate: ErrorExpectation, + sessions: Vec, + ) -> Self { + let soak_requests = + sessions + .iter() + .flat_map(|session| { + session.payloads.iter().cloned().map(|body| { + RequestCase::new(Endpoint::Chat, session.session_id.clone(), body) + }) + }) + .collect(); + Self { + id, + group, + description, + expected, + expected_error_rate, + sessions, + soak_requests, + load_profiles: fixed_load(), + } + } + + /// Return one prebuilt request with only its streaming flag changed. + pub fn request(&self, index: usize, stream: bool) -> PreparedRequest<'_> { + let selected = &self.soak_requests[index % self.soak_requests.len()]; + PreparedRequest { + endpoint: selected.endpoint, + session_id: &selected.session_id, + body: if stream { + &selected.streaming_body + } else { + &selected.nonstreaming_body + }, + } + } +} + +/// Return the one fixed schedule used by ordinary request scenarios. +pub fn fixed_load() -> Vec { + vec![LoadProfile::Fixed { id: "fixed" }] +} + +/// Build the three bounded load schedules used for the short baseline. +pub fn baseline_load() -> Vec { + vec![ + LoadProfile::Fixed { id: "fixed" }, + LoadProfile::ConcurrencyKnee { + id: "concurrency-knee", + concurrency_steps: vec![1, 4, 16, 64, 128], + }, + LoadProfile::TrafficBurst { + id: "traffic-burst", + duration_seconds: 30, + points: vec![ + RatePoint { + time_s: 0, + rate_multiplier: 1.0, + }, + RatePoint { + time_s: 10, + rate_multiplier: 1.0, + }, + RatePoint { + time_s: 11, + rate_multiplier: 10.0, + }, + RatePoint { + time_s: 16, + rate_multiplier: 10.0, + }, + RatePoint { + time_s: 17, + rate_multiplier: 1.0, + }, + RatePoint { + time_s: 30, + rate_multiplier: 1.0, + }, + ], + }, + ] +} + +/// Build one raw Chat Completions payload. +pub fn chat_payload(model: &str, messages: Vec, max_tokens: u32) -> Value { + json!({ + "model": model, + "messages": messages, + "max_tokens": max_tokens, + "temperature": 0, + "stream": true, + }) +} + +/// Build one user message. +pub fn user(content: impl Into) -> Value { + json!({"role": "user", "content": content.into()}) +} + +/// Build one assistant message. +pub fn assistant(content: impl Into) -> Value { + json!({"role": "assistant", "content": content.into()}) +} + +/// Build repeated context with a stable approximate token count. +pub fn token_text(tokens: usize, label: &str) -> String { + format!("[scenario:{label}] {}", "x ".repeat(tokens)) +} + +/// Build the complete catalog in stable report order. +pub fn catalog(options: ScenarioOptions<'_>) -> Result, String> { + options.validate()?; + Ok(vec![ + short_interactive::build(options), + long_context::build(options), + decode_heavy::build(options), + prefix_reuse::build(options), + mixed_traffic::build(options), + growing_conversation::build(options), + large_tool_catalog::build(options), + tool_call_burst::build(options), + stage_transitions::build(options), + classifier_mix::build(options), + context_overflow::build(options), + failure_pressure::build(options), + client_cancellation::build(options), + ]) +} + +/// Select explicit scenario IDs or every scenario in a named set. +pub fn select( + options: ScenarioOptions<'_>, + set: ScenarioSet, + requested: &[String], +) -> Result, String> { + let mut catalog = catalog(options)?; + if requested.is_empty() { + return Ok(catalog + .into_iter() + .filter(|scenario| set.contains(scenario.group)) + .collect()); + } + + let mut seen = HashSet::new(); + let mut selected = Vec::new(); + for id in requested { + if !seen.insert(id) { + return Err(format!("scenario {id:?} was requested more than once")); + } + let index = catalog + .iter() + .position(|scenario| scenario.id == id) + .ok_or_else(|| format!("unknown scenario {id:?}"))?; + selected.push(catalog.remove(index)); + } + Ok(selected) +} + +#[derive(Serialize)] +struct InputsFile<'a> { + data: &'a [Session], +} + +#[derive(Serialize)] +struct Manifest<'a> { + schema_version: u32, + model: &'a str, + scenarios: Vec>, +} + +#[derive(Serialize)] +struct ManifestScenario<'a> { + id: &'a str, + group: ScenarioGroup, + description: &'a str, + expected: &'a str, + expected_error_rate: ErrorExpectation, + input_file: String, + load_profiles: &'a [LoadProfile], +} + +/// Export the selected scenario catalog in AIPerf's verbatim inputs JSON format. +pub fn export(scenarios: &[Scenario], model: &str, output_dir: &Path) -> Result { + if output_dir.exists() { + return Err(format!( + "scenario output directory already exists: {}", + output_dir.display() + )); + } + let inputs_dir = output_dir.join("inputs"); + fs::create_dir_all(&inputs_dir).map_err(|error| error.to_string())?; + + let mut entries = Vec::new(); + for scenario in scenarios { + let relative = format!("inputs/{}.json", scenario.id); + let path = output_dir.join(&relative); + let body = serde_json::to_string_pretty(&InputsFile { + data: &scenario.sessions, + }) + .map_err(|error| error.to_string())?; + fs::write(path, format!("{body}\n")).map_err(|error| error.to_string())?; + entries.push(ManifestScenario { + id: scenario.id, + group: scenario.group, + description: scenario.description, + expected: scenario.expected, + expected_error_rate: scenario.expected_error_rate, + input_file: relative, + load_profiles: &scenario.load_profiles, + }); + } + + let manifest_path = output_dir.join("manifest.json"); + let body = serde_json::to_string_pretty(&Manifest { + schema_version: 1, + model, + scenarios: entries, + }) + .map_err(|error| error.to_string())?; + fs::write(&manifest_path, format!("{body}\n")).map_err(|error| error.to_string())?; + Ok(manifest_path) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn options() -> ScenarioOptions<'static> { + ScenarioOptions { + model: "switchyard/test", + prompt_bytes: 128, + max_output_tokens: 32, + context_window_tokens: 16_384, + } + } + + #[test] + fn catalog_is_unique_bounded_and_serializable() -> Result<(), String> { + let scenarios = catalog(options())?; + let names = scenarios + .iter() + .map(|scenario| scenario.id) + .collect::>(); + assert_eq!(names.len(), scenarios.len()); + assert!(scenarios.iter().all(|scenario| { + !scenario.sessions.is_empty() + && !scenario.soak_requests.is_empty() + && scenario.sessions.len() <= 32 + && scenario + .sessions + .iter() + .all(|session| !session.payloads.is_empty() && session.payloads.len() <= 32) + })); + + let parent = tempfile::tempdir().map_err(|error| error.to_string())?; + let output = parent.path().join("export"); + let manifest = export(&scenarios, options().model, &output)?; + let payload: Value = + serde_json::from_str(&fs::read_to_string(manifest).map_err(|error| error.to_string())?) + .map_err(|error| error.to_string())?; + assert_eq!(payload["scenarios"].as_array().map(Vec::len), Some(13)); + Ok(()) + } + + #[test] + fn explicit_selection_keeps_order_and_rejects_bad_names() -> Result<(), String> { + let requested = vec!["tool-call-burst".to_string(), "long-context".to_string()]; + let selected = select(options(), ScenarioSet::Core, &requested)?; + assert_eq!( + selected + .iter() + .map(|scenario| scenario.id) + .collect::>(), + ["tool-call-burst", "long-context"] + ); + assert!(select(options(), ScenarioSet::Core, &["missing".to_string()]).is_err()); + Ok(()) + } + + #[test] + fn agentic_payloads_keep_their_bounded_contracts() -> Result<(), String> { + let scenarios = catalog(options())?; + let find = |id| { + scenarios + .iter() + .find(|scenario| scenario.id == id) + .ok_or_else(|| format!("missing scenario {id}")) + }; + + let long = find("long-context")?; + assert!(long.sessions.iter().any(|session| { + session.payloads[0]["messages"][0]["content"] + .as_str() + .is_some_and(|content| content.len() > 8_000) + })); + + let tools = find("large-tool-catalog")?; + assert_eq!( + tools.sessions[1].payloads[0]["tools"] + .as_array() + .map(Vec::len), + Some(64) + ); + + let burst = find("tool-call-burst")?; + assert_eq!(burst.sessions[0].payloads.len(), 9); + let final_messages = burst.sessions[0].payloads[8]["messages"] + .as_array() + .ok_or_else(|| "tool-call burst messages were not an array".to_string())?; + assert!( + final_messages.iter().any(|message| { + message["role"] == "tool" && message["tool_call_id"] == "call_7" + }) + ); + + let growing = find("growing-conversation")?; + let message_counts = growing.sessions[0] + .payloads + .iter() + .map(|payload| payload["messages"].as_array().map(Vec::len).unwrap_or(0)) + .collect::>(); + assert!(message_counts.windows(2).all(|pair| pair[0] < pair[1])); + assert!(find("failure-pressure")?.expected_error_rate.min_rate > 0.0); + Ok(()) + } +} diff --git a/crates/switchyard-soak/src/scenarios/prefix_reuse.rs b/crates/switchyard-soak/src/scenarios/prefix_reuse.rs new file mode 100644 index 000000000..7272ec501 --- /dev/null +++ b/crates/switchyard-soak/src/scenarios/prefix_reuse.rs @@ -0,0 +1,37 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use super::{ + ErrorExpectation, Scenario, ScenarioGroup, ScenarioOptions, Session, chat_payload, token_text, + user, +}; + +pub fn build(options: ScenarioOptions<'_>) -> Scenario { + let prefix_tokens = (options.prompt_bytes / 2).clamp(512, 8_192); + let shared = token_text(prefix_tokens, "prefix_reuse_shared"); + let sessions = (0..8) + .map(|index| { + let prefix = if index < 4 { + shared.clone() + } else { + token_text(prefix_tokens, &format!("prefix_reuse_unique_{index}")) + }; + Session { + session_id: format!("prefix-reuse-{index}"), + payloads: vec![chat_payload( + options.model, + vec![user(format!("{prefix}\nReturn item {index}."))], + options.max_output_tokens.min(32), + )], + } + }) + .collect(); + Scenario::chat( + "prefix-reuse", + ScenarioGroup::Core, + "Matched requests with shared and unique long prefixes.", + "The report exposes cache-sensitive TTFT without changing route behavior.", + ErrorExpectation::SUCCESS, + sessions, + ) +} diff --git a/crates/switchyard-soak/src/scenarios/short_interactive.rs b/crates/switchyard-soak/src/scenarios/short_interactive.rs new file mode 100644 index 000000000..023f04161 --- /dev/null +++ b/crates/switchyard-soak/src/scenarios/short_interactive.rs @@ -0,0 +1,61 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use crate::client::{Endpoint, request_body}; + +use super::{ + ErrorExpectation, RequestCase, Scenario, ScenarioGroup, ScenarioOptions, Session, + baseline_load, chat_payload, user, +}; + +pub fn build(options: ScenarioOptions<'_>) -> Scenario { + let prompts = [ + "Reply with exactly OK.".to_string(), + "Name one primary color.".to_string(), + "Return the number after 41.".to_string(), + format!( + "[scenario:short_interactive] {} Reply briefly.", + "stable prefix ".repeat(options.prompt_bytes.div_ceil(14)) + ), + ]; + let sessions = prompts + .iter() + .enumerate() + .map(|(index, prompt)| Session { + session_id: format!("short-{index}"), + payloads: vec![chat_payload( + options.model, + vec![user(prompt)], + options.max_output_tokens, + )], + }) + .collect(); + let soak_requests = Endpoint::ALL + .iter() + .flat_map(|endpoint| { + prompts.iter().enumerate().map(move |(index, prompt)| { + RequestCase::new( + *endpoint, + format!("short-{index}-{}", endpoint.as_str()), + request_body( + *endpoint, + options.model, + prompt, + options.max_output_tokens, + true, + ), + ) + }) + }) + .collect(); + Scenario { + id: "short-interactive", + group: ScenarioGroup::Core, + description: "Short prompts that establish HTTP, routing, TTFT, and latency overhead.", + expected: "All public endpoints succeed; oha and AIPerf establish the baseline.", + expected_error_rate: ErrorExpectation::SUCCESS, + sessions, + soak_requests, + load_profiles: baseline_load(), + } +} diff --git a/crates/switchyard-soak/src/scenarios/stage_transitions.rs b/crates/switchyard-soak/src/scenarios/stage_transitions.rs new file mode 100644 index 000000000..63916e37e --- /dev/null +++ b/crates/switchyard-soak/src/scenarios/stage_transitions.rs @@ -0,0 +1,53 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use serde_json::json; + +use super::{ + ErrorExpectation, Scenario, ScenarioGroup, ScenarioOptions, Session, chat_payload, user, +}; + +pub fn build(options: ScenarioOptions<'_>) -> Scenario { + let mut messages = vec![user( + "[scenario:stage_exploration] Find the failing module, then fix it.", + )]; + let mut payloads = vec![chat_payload( + options.model, + messages.clone(), + options.max_output_tokens.min(128), + )]; + + messages.extend([ + json!({"role":"assistant","tool_calls":[{"id":"stage_1","type":"function","function":{"name":"run_tests","arguments":"{}"}}]}), + json!({"role":"tool","tool_call_id":"stage_1","content":"fatal: critical tool failure: repository index is unavailable"}), + ]); + payloads.push(chat_payload( + options.model, + messages.clone(), + options.max_output_tokens.min(128), + )); + + messages.extend([ + json!({"role":"assistant","tool_calls":[{"id":"stage_2","type":"function","function":{"name":"apply_patch","arguments":"{}"}}]}), + json!({"role":"tool","tool_call_id":"stage_2","content":"patch applied"}), + json!({"role":"assistant","tool_calls":[{"id":"stage_3","type":"function","function":{"name":"run_tests","arguments":"{}"}}]}), + json!({"role":"tool","tool_call_id":"stage_3","content":"all tests passed"}), + ]); + payloads.push(chat_payload( + options.model, + messages, + options.max_output_tokens.min(128), + )); + + Scenario::chat( + "stage-transitions", + ScenarioGroup::Agentic, + "One growing session that moves through exploration, critical failure, and productive work.", + "The stage router changes tiers at the configured evidence boundaries.", + ErrorExpectation::SUCCESS, + vec![Session { + session_id: "stage-transitions".to_string(), + payloads, + }], + ) +} diff --git a/crates/switchyard-soak/src/scenarios/tool_call_burst.rs b/crates/switchyard-soak/src/scenarios/tool_call_burst.rs new file mode 100644 index 000000000..1c6b35d69 --- /dev/null +++ b/crates/switchyard-soak/src/scenarios/tool_call_burst.rs @@ -0,0 +1,65 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use serde_json::{Value, json}; + +use super::{ + ErrorExpectation, Scenario, ScenarioGroup, ScenarioOptions, Session, chat_payload, user, +}; + +pub fn build(options: ScenarioOptions<'_>) -> Scenario { + let mut messages = vec![user( + "[scenario:tool_call_burst] Inspect eight shards, one call at a time.", + )]; + let mut payloads = vec![chat_payload( + options.model, + messages.clone(), + options.max_output_tokens.min(128), + )]; + for index in 0..8 { + let call_id = format!("call_{index}"); + messages.push(json!({ + "role": "assistant", + "content": null, + "tool_calls": [{ + "id": call_id, + "type": "function", + "function": {"name": "inspect_shard", "arguments": format!("{{\"shard\":{index}}}")} + }] + })); + messages.push(json!({ + "role": "tool", + "tool_call_id": format!("call_{index}"), + "content": format!("shard {index}: healthy") + })); + payloads.push(chat_payload( + options.model, + messages.clone(), + options.max_output_tokens.min(128), + )); + } + for payload in &mut payloads { + payload["tools"] = Value::Array(vec![json!({ + "type": "function", + "function": { + "name": "inspect_shard", + "parameters": { + "type": "object", + "properties": {"shard": {"type": "integer"}}, + "required": ["shard"] + } + } + })]); + } + Scenario::chat( + "tool-call-burst", + ScenarioGroup::Agentic, + "An eight-turn burst of linked assistant tool calls and tool results.", + "The route keeps session state and forwards every call/result pair.", + ErrorExpectation::SUCCESS, + vec![Session { + session_id: "tool-call-burst".to_string(), + payloads, + }], + ) +} diff --git a/crates/switchyard-soak/src/stats.rs b/crates/switchyard-soak/src/stats.rs new file mode 100644 index 000000000..dcb8f1f20 --- /dev/null +++ b/crates/switchyard-soak/src/stats.rs @@ -0,0 +1,453 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Bounded in-memory run state, latency percentiles, and the final pass/fail summary. + +use std::collections::BTreeMap; +use std::time::SystemTime; + +use rand::rngs::StdRng; +use rand::{RngExt, SeedableRng}; +use serde::Serialize; + +/// Cap on retained latency samples; a long run stays within this bound by reservoir sampling. +const RESERVOIR_SIZE: usize = 100_000; + +/// Round to three decimals so result files stay readable. +pub fn round3(value: f64) -> f64 { + (value * 1000.0).round() / 1000.0 +} + +/// Parse a duration such as `30s`, `15m`, or `48h` into seconds. +pub fn parse_duration(value: &str) -> Result { + let text = value.trim().to_lowercase(); + let bad = || "duration must use s, m, or h, for example 30s or 48h".to_string(); + let unit = text.chars().last().ok_or_else(bad)?; + let multiplier = match unit { + 's' => 1.0, + 'm' => 60.0, + 'h' => 3600.0, + _ => return Err(bad()), + }; + let number = &text[..text.len() - unit.len_utf8()]; + let seconds = number.parse::().map_err(|_| bad())? * multiplier; + if !seconds.is_finite() || seconds <= 0.0 { + return Err("duration must be greater than zero".to_string()); + } + Ok(seconds) +} + +#[derive(Default)] +pub struct LatencyStats { + pub p50_ms: Option, + pub p95_ms: Option, + pub p99_ms: Option, + pub max_ms: Option, +} + +fn percentile_sorted(values: &[f64], quantile: f64) -> Option { + let index = (((values.len().checked_sub(1)?) as f64) * quantile).round_ties_even() as usize; + Some(round3(values[index])) +} + +pub fn latency_stats(values: &mut [f64]) -> LatencyStats { + values.sort_by(|a, b| a.total_cmp(b)); + LatencyStats { + p50_ms: percentile_sorted(values, 0.50), + p95_ms: percentile_sorted(values, 0.95), + p99_ms: percentile_sorted(values, 0.99), + max_ms: values.last().copied().map(round3), + } +} + +/// ISO-8601 UTC timestamp at second precision, e.g. `2026-07-30T18:04:00Z`. +pub fn now_utc_string() -> String { + humantime::format_rfc3339_seconds(SystemTime::now()).to_string() +} + +/// Compact UTC stamp for a results directory, e.g. `20260730T180400Z`. +pub fn utc_dir_stamp() -> String { + now_utc_string() + .chars() + .filter(|character| !matches!(character, '-' | ':')) + .collect() +} + +/// Request results collected since the previous report. +#[derive(Default)] +pub struct IntervalStats { + pub successes: u64, + pub failures: u64, + pub latencies_ms: Vec, +} + +/// Bounded in-memory state for one soak run. +pub struct RunStats { + rng: StdRng, + interval: IntervalStats, + pub total_successes: u64, + pub total_failures: u64, + pub endpoint_successes: BTreeMap, + pub endpoint_failures: BTreeMap, + pub scenario_successes: BTreeMap, + pub scenario_failures: BTreeMap, + pub error_kinds: BTreeMap, + latency_reservoir: Vec, + latency_count: u64, + pub health_checks: u64, + pub health_failures: u64, + pub metrics_checks: u64, + pub metrics_failures: u64, + pub process_checks: u64, + pub process_failures: u64, + pub canaries: u64, + pub canary_failures: u64, + pub server_restarts: u64, + pub previous_server_requests: Option, + pub rss_samples: Vec, + pub completed_duration: bool, +} + +impl RunStats { + pub fn new(seed: u64) -> Self { + Self { + rng: StdRng::seed_from_u64(seed), + interval: IntervalStats::default(), + total_successes: 0, + total_failures: 0, + endpoint_successes: BTreeMap::new(), + endpoint_failures: BTreeMap::new(), + scenario_successes: BTreeMap::new(), + scenario_failures: BTreeMap::new(), + error_kinds: BTreeMap::new(), + latency_reservoir: Vec::new(), + latency_count: 0, + health_checks: 0, + health_failures: 0, + metrics_checks: 0, + metrics_failures: 0, + process_checks: 0, + process_failures: 0, + canaries: 0, + canary_failures: 0, + server_restarts: 0, + previous_server_requests: None, + rss_samples: Vec::new(), + completed_duration: false, + } + } + + /// Record one completed inference request; `error_kind` is `None` on success. + pub fn record( + &mut self, + endpoint: &str, + scenario: &str, + latency_ms: f64, + error_kind: Option<&str>, + ) { + match error_kind { + None => { + self.interval.successes += 1; + self.total_successes += 1; + *self + .endpoint_successes + .entry(endpoint.to_string()) + .or_default() += 1; + *self + .scenario_successes + .entry(scenario.to_string()) + .or_default() += 1; + } + Some(kind) => { + self.interval.failures += 1; + self.total_failures += 1; + *self + .endpoint_failures + .entry(endpoint.to_string()) + .or_default() += 1; + *self + .scenario_failures + .entry(scenario.to_string()) + .or_default() += 1; + *self.error_kinds.entry(kind.to_string()).or_default() += 1; + } + } + self.interval.latencies_ms.push(latency_ms); + self.latency_count += 1; + if self.latency_reservoir.len() < RESERVOIR_SIZE { + self.latency_reservoir.push(latency_ms); + } else { + let index = self.rng.random_range(0..self.latency_count) as usize; + if index < RESERVOIR_SIZE { + self.latency_reservoir[index] = latency_ms; + } + } + } + + /// Return and reset the current interval. + pub fn take_interval(&mut self) -> IntervalStats { + std::mem::take(&mut self.interval) + } +} + +/// Final result written to `summary.json`. +#[derive(Serialize)] +pub struct Summary { + pub passed: bool, + pub failure_reasons: Vec, + pub completed_duration: bool, + pub elapsed_seconds: f64, + pub requests: u64, + pub successes: u64, + pub failures: u64, + pub error_rate: f64, + pub requests_per_second: f64, + pub endpoint_successes: BTreeMap, + pub endpoint_failures: BTreeMap, + pub scenario_successes: BTreeMap, + pub scenario_failures: BTreeMap, + pub error_kinds: BTreeMap, + pub latency_p50_ms: Option, + pub latency_p95_ms: Option, + pub latency_p99_ms: Option, + pub health_checks: u64, + pub health_failures: u64, + pub metrics_checks: u64, + pub metrics_failures: u64, + pub process_checks: u64, + pub process_failures: u64, + pub invalid_request_canaries: u64, + pub invalid_request_canary_failures: u64, + pub detected_server_restarts: u64, + pub rss_first_mib: Option, + pub rss_last_mib: Option, + pub rss_max_mib: Option, + pub rss_growth_mib: Option, + pub error_records: u64, + pub dropped_error_records: u64, +} + +/// Build the final result and its release-gate reasons. +pub fn build_summary( + stats: &RunStats, + elapsed_seconds: f64, + max_error_rate: f64, + max_rss_growth_mib: Option, + error_records: u64, + dropped_error_records: u64, + task_failures: &[String], +) -> Summary { + let total = stats.total_successes + stats.total_failures; + let error_rate = if total > 0 { + stats.total_failures as f64 / total as f64 + } else { + 1.0 + }; + let rss_first = stats.rss_samples.first().copied(); + let rss_last = stats.rss_samples.last().copied(); + let rss_growth = match (rss_first, rss_last) { + (Some(first), Some(last)) => Some(last - first), + _ => None, + }; + + let mut reasons: Vec = task_failures.to_vec(); + if !stats.completed_duration { + reasons.push("the run stopped before the requested duration".to_string()); + } + if total == 0 { + reasons.push("no inference requests completed".to_string()); + } + if error_rate > max_error_rate { + reasons.push(format!( + "request error rate {:.4}% exceeded the {:.4}% limit", + error_rate * 100.0, + max_error_rate * 100.0 + )); + } + if stats.health_failures > 0 { + reasons.push(format!("{} liveness checks failed", stats.health_failures)); + } + if stats.metrics_failures > 0 { + reasons.push(format!( + "{} server metrics checks failed", + stats.metrics_failures + )); + } + if stats.process_failures > 0 { + reasons.push(format!( + "{} server process checks failed", + stats.process_failures + )); + } + if stats.canary_failures > 0 { + reasons.push(format!( + "{} invalid-request recovery checks failed", + stats.canary_failures + )); + } + if stats.server_restarts > 0 { + reasons.push(format!( + "server counters reset {} time(s)", + stats.server_restarts + )); + } + if let (Some(limit), Some(growth)) = (max_rss_growth_mib, rss_growth) + && growth > limit + { + reasons.push(format!( + "server RSS grew {growth:.1} MiB, above the {limit:.1} MiB limit" + )); + } + + let rss_max = stats + .rss_samples + .iter() + .copied() + .fold(None, |acc: Option, value| { + Some(acc.map_or(value, |m| m.max(value))) + }); + let requests_per_second = if elapsed_seconds > 0.0 { + total as f64 / elapsed_seconds + } else { + 0.0 + }; + + let mut latency_samples = stats.latency_reservoir.clone(); + let latency = latency_stats(&mut latency_samples); + + Summary { + passed: reasons.is_empty(), + failure_reasons: reasons, + completed_duration: stats.completed_duration, + elapsed_seconds: round3(elapsed_seconds), + requests: total, + successes: stats.total_successes, + failures: stats.total_failures, + error_rate, + requests_per_second, + endpoint_successes: stats.endpoint_successes.clone(), + endpoint_failures: stats.endpoint_failures.clone(), + scenario_successes: stats.scenario_successes.clone(), + scenario_failures: stats.scenario_failures.clone(), + error_kinds: stats.error_kinds.clone(), + latency_p50_ms: latency.p50_ms, + latency_p95_ms: latency.p95_ms, + latency_p99_ms: latency.p99_ms, + health_checks: stats.health_checks, + health_failures: stats.health_failures, + metrics_checks: stats.metrics_checks, + metrics_failures: stats.metrics_failures, + process_checks: stats.process_checks, + process_failures: stats.process_failures, + invalid_request_canaries: stats.canaries, + invalid_request_canary_failures: stats.canary_failures, + detected_server_restarts: stats.server_restarts, + rss_first_mib: rss_first, + rss_last_mib: rss_last, + rss_max_mib: rss_max, + rss_growth_mib: rss_growth, + error_records, + dropped_error_records, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_duration_accepts_units_and_rejects_invalid_values() { + assert_eq!(parse_duration("30s"), Ok(30.0)); + assert_eq!(parse_duration(".5s"), Ok(0.5)); + assert_eq!(parse_duration("2.5m"), Ok(150.0)); + assert_eq!(parse_duration("48h"), Ok(172_800.0)); + for value in ["0s", "10", "5x", "-3s", "1d", "NaNs", "infs"] { + assert!(parse_duration(value).is_err(), "{value} should be rejected"); + } + } + + #[test] + fn latency_stats_sorts_once_and_handles_empty_input() { + let mut values = [60.0, 10.0, 50.0, 20.0, 40.0, 30.0]; + let latency = latency_stats(&mut values); + assert_eq!(latency.p50_ms, Some(30.0)); + assert_eq!(latency.p95_ms, Some(60.0)); + assert_eq!(latency.p99_ms, Some(60.0)); + assert_eq!(latency.max_ms, Some(60.0)); + + let empty = latency_stats(&mut []); + assert_eq!(empty.p50_ms, None); + assert_eq!(empty.max_ms, None); + } + + #[test] + fn record_tracks_interval_and_cumulative_results() { + let mut stats = RunStats::new(1); + stats.record("chat", "short-interactive", 10.0, None); + stats.record("messages", "long-context", 20.0, Some("timeout")); + + let interval = stats.take_interval(); + assert_eq!(interval.successes, 1); + assert_eq!(interval.failures, 1); + assert_eq!(stats.total_successes, 1); + assert_eq!(stats.total_failures, 1); + assert_eq!(stats.error_kinds.get("timeout"), Some(&1)); + } + + #[test] + fn latency_samples_stay_bounded_after_the_reservoir_fills() { + let mut stats = RunStats::new(1); + for sample in 0..=RESERVOIR_SIZE { + stats.record("chat", "short-interactive", sample as f64, None); + } + + assert_eq!(stats.latency_reservoir.len(), RESERVOIR_SIZE); + assert_eq!(stats.latency_count, RESERVOIR_SIZE as u64 + 1); + } + + #[test] + fn summary_reports_all_failed_gates() { + let mut stats = RunStats::new(1); + stats.total_successes = 998; + stats.total_failures = 2; + stats.health_failures = 1; + stats.metrics_failures = 1; + stats.process_failures = 1; + stats.canary_failures = 1; + stats.server_restarts = 1; + stats.rss_samples = vec![100.0, 700.0]; + + let summary = build_summary( + &stats, + 10.0, + 0.001, + Some(512.0), + 0, + 0, + &["reporter failed: boom".to_string()], + ); + + assert!(!summary.passed); + for reason in [ + "run stopped", + "error rate", + "liveness", + "metrics", + "process", + "invalid-request", + "counters reset", + "RSS grew", + "reporter failed", + ] { + assert!( + summary + .failure_reasons + .iter() + .any(|message| message.contains(reason)), + "missing {reason:?}: {:?}", + summary.failure_reasons, + ); + } + assert_eq!(summary.rss_growth_mib, Some(600.0)); + } +} diff --git a/crates/switchyard-soak/tests/soak.rs b/crates/switchyard-soak/tests/soak.rs new file mode 100644 index 000000000..26c8dcbda --- /dev/null +++ b/crates/switchyard-soak/tests/soak.rs @@ -0,0 +1,331 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! End-to-end tests against a local server that implements the public Switchyard APIs. + +use std::collections::HashSet; +use std::error::Error; +use std::sync::Arc; + +use axum::Router; +use axum::extract::{Json, State}; +use axum::http::{HeaderMap, StatusCode, header}; +use axum::response::{IntoResponse, Response}; +use axum::routing::{get, post}; +use clap::Parser; +use parking_lot::Mutex; +use serde_json::{Value, json}; + +use switchyard_soak::{Args, run}; + +type TestResult = Result<(), Box>; + +struct TestServer { + base_url: String, + task: tokio::task::JoinHandle<()>, +} + +impl Drop for TestServer { + fn drop(&mut self) { + self.task.abort(); + } +} + +async fn serve(app: Router) -> Result> { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let addr = listener.local_addr()?; + let task = tokio::spawn(async move { + let _ = axum::serve(listener, app.into_make_service()).await; + }); + Ok(TestServer { + base_url: format!("http://{addr}"), + task, + }) +} + +#[derive(Clone, Default)] +struct MockState { + is_chat_broken: bool, + seen: Arc>>, +} + +impl MockState { + fn record(&self, endpoint: &str, body: &Value) { + let stream = body.get("stream").and_then(Value::as_bool) == Some(true); + self.seen.lock().insert(format!("{endpoint}:{stream}")); + } +} + +fn switchyard(is_chat_broken: bool) -> (Router, Arc>>) { + async fn health() -> Response { + Json(json!({"status": "ok"})).into_response() + } + async fn metrics() -> Response { + "switchyard_total_requests 1\nswitchyard_total_errors 0\n".into_response() + } + async fn models() -> Response { + Json(json!({"data": [{"id": "soak-route"}]})).into_response() + } + async fn chat( + State(state): State, + headers: HeaderMap, + Json(body): Json, + ) -> Response { + let invalid_messages = body.get("messages").is_some_and(|messages| { + messages + .as_array() + .is_none_or(|messages| messages.is_empty()) + }); + if invalid_messages { + return if state.is_chat_broken { + Json(json!({"choices": []})).into_response() + } else { + (StatusCode::BAD_REQUEST, "messages must not be empty").into_response() + }; + } + if !headers.contains_key("x-switchyard-session-id") { + return (StatusCode::BAD_REQUEST, "missing session id").into_response(); + } + state.record("chat", &body); + if state.is_chat_broken { + return if body.get("stream") == Some(&Value::Bool(true)) { + Json(json!({"choices": []})).into_response() + } else { + Json(json!({})).into_response() + }; + } + response(&body, "choices") + } + async fn messages( + State(state): State, + headers: HeaderMap, + Json(body): Json, + ) -> Response { + if !headers.contains_key("x-switchyard-session-id") { + return (StatusCode::BAD_REQUEST, "missing session id").into_response(); + } + state.record("messages", &body); + response(&body, "content") + } + async fn responses( + State(state): State, + headers: HeaderMap, + Json(body): Json, + ) -> Response { + if !headers.contains_key("x-switchyard-session-id") { + return (StatusCode::BAD_REQUEST, "missing session id").into_response(); + } + state.record("responses", &body); + response(&body, "output") + } + fn response(body: &Value, field: &str) -> Response { + if body.get("stream") == Some(&Value::Bool(true)) { + let stream = match field { + "choices" => "data: {\"choices\":[]}\n\ndata: [DONE]\n\n", + "content" => "event: message_stop\ndata: {\"type\":\"message_stop\"}\n\n", + "output" => { + "event: response.completed\ndata: {\"type\":\"response.completed\"}\n\n" + } + _ => "", + }; + ([(header::CONTENT_TYPE, "text/event-stream")], stream).into_response() + } else { + let mut response = serde_json::Map::new(); + response.insert(field.to_string(), json!([])); + Json(Value::Object(response)).into_response() + } + } + + let state = MockState { + is_chat_broken, + ..MockState::default() + }; + let seen = state.seen.clone(); + let app = Router::new() + .route("/health", get(health)) + .route("/metrics", get(metrics)) + .route("/v1/models", get(models)) + .route("/v1/chat/completions", post(chat)) + .route("/v1/messages", post(messages)) + .route("/v1/responses", post(responses)) + .with_state(state); + (app, seen) +} + +fn args(base_url: &str, results_dir: &str) -> Result { + Args::try_parse_from([ + "switchyard-soak", + "--base-url", + base_url, + "--model", + "soak-route", + "--duration", + "0.8s", + "--concurrency", + "2", + "--report-interval", + "0.1", + "--invalid-canary-interval", + "0.1", + "--scenario", + "short-interactive", + "--results-dir", + results_dir, + ]) +} + +#[tokio::test] +async fn short_run_exercises_every_response_variant_and_passes() -> TestResult { + let (app, seen) = switchyard(false); + let server = serve(app).await?; + let dir = tempfile::tempdir()?; + let results_dir = dir.path().join("soak-results"); + let args = args( + &server.base_url, + results_dir.to_str().ok_or("non-utf8 results dir")?, + )?; + + args.validate()?; + assert_eq!(run(args).await?, 0); + + let summary: Value = + serde_json::from_str(&std::fs::read_to_string(results_dir.join("summary.json"))?)?; + assert_eq!(summary["passed"], json!(true)); + assert!(summary["invalid_request_canaries"].as_u64().unwrap_or(0) > 0); + assert_eq!( + *seen.lock(), + HashSet::from([ + "chat:false".to_string(), + "chat:true".to_string(), + "messages:false".to_string(), + "messages:true".to_string(), + "responses:false".to_string(), + "responses:true".to_string(), + ]) + ); + Ok(()) +} + +#[tokio::test] +async fn bad_responses_and_canary_write_a_failing_summary() -> TestResult { + let (app, _) = switchyard(true); + let server = serve(app).await?; + let dir = tempfile::tempdir()?; + let results_dir = dir.path().join("soak-results"); + let args = args( + &server.base_url, + results_dir.to_str().ok_or("non-utf8 results dir")?, + )?; + + assert_eq!(run(args).await?, 1); + + let summary: Value = + serde_json::from_str(&std::fs::read_to_string(results_dir.join("summary.json"))?)?; + assert_eq!(summary["passed"], json!(false)); + assert!( + summary["error_kinds"]["invalid_response"] + .as_u64() + .unwrap_or(0) + > 0 + ); + assert!( + summary["error_kinds"]["invalid_stream"] + .as_u64() + .unwrap_or(0) + > 0 + ); + assert!( + summary["invalid_request_canary_failures"] + .as_u64() + .unwrap_or(0) + > 0 + ); + Ok(()) +} + +#[tokio::test] +async fn nonbaseline_scenarios_all_cross_the_public_request_path() -> TestResult { + let (app, _) = switchyard(false); + let server = serve(app).await?; + let dir = tempfile::tempdir()?; + let results_dir = dir.path().join("scenario-results"); + let args = Args::try_parse_from([ + "switchyard-soak", + "--base-url", + &server.base_url, + "--model", + "soak-route", + "--duration", + "0.8s", + "--concurrency", + "4", + "--report-interval", + "0.2", + "--invalid-canary-interval", + "0", + "--scenario", + "long-context", + "--scenario", + "decode-heavy", + "--scenario", + "prefix-reuse", + "--scenario", + "mixed-traffic", + "--scenario", + "growing-conversation", + "--scenario", + "large-tool-catalog", + "--scenario", + "tool-call-burst", + "--scenario", + "stage-transitions", + "--scenario", + "classifier-mix", + "--results-dir", + results_dir.to_str().ok_or("non-utf8 results dir")?, + ])?; + + assert_eq!(run(args).await?, 0); + let summary: Value = + serde_json::from_str(&std::fs::read_to_string(results_dir.join("summary.json"))?)?; + let successes = summary["scenario_successes"] + .as_object() + .ok_or("scenario_successes was not an object")?; + for scenario in [ + "long-context", + "decode-heavy", + "prefix-reuse", + "mixed-traffic", + "growing-conversation", + "large-tool-catalog", + "tool-call-burst", + "stage-transitions", + "classifier-mix", + ] { + assert!(successes.contains_key(scenario), "missing {scenario}"); + } + Ok(()) +} + +#[tokio::test] +async fn unknown_model_is_rejected_before_the_run_starts() -> TestResult { + let (app, _) = switchyard(false); + let server = serve(app).await?; + let dir = tempfile::tempdir()?; + let results_dir = dir.path().join("soak-results"); + let args = Args::try_parse_from([ + "switchyard-soak", + "--base-url", + &server.base_url, + "--model", + "missing", + "--duration", + "1s", + "--results-dir", + results_dir.to_str().ok_or("non-utf8 results dir")?, + ])?; + + let error = run(args).await.unwrap_err(); + assert!(error.contains("is not listed"), "{error}"); + Ok(()) +} diff --git a/crates/switchyard-translation/src/codecs/anthropic/buffered.rs b/crates/switchyard-translation/src/codecs/anthropic/buffered.rs index 87dc58b6c..23fd5140c 100644 --- a/crates/switchyard-translation/src/codecs/anthropic/buffered.rs +++ b/crates/switchyard-translation/src/codecs/anthropic/buffered.rs @@ -91,7 +91,13 @@ impl FormatCodec for AnthropicMessagesCodec { content, }); } - if let Some(messages) = body.get("messages").and_then(Value::as_array) { + if let Some(messages) = body.get("messages") { + let messages = messages + .as_array() + .ok_or_else(|| TranslationError::InvalidType { + path: "$.messages".to_string(), + expected: "array", + })?; let mut generated_id = 0; for (index, message) in messages.iter().enumerate() { let Some(message) = message.as_object() else { diff --git a/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs b/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs index d829ca316..4b0394f43 100644 --- a/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs +++ b/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs @@ -72,7 +72,13 @@ impl FormatCodec for OpenAiChatCodec { ..LlmRequest::default() }; - if let Some(messages) = body.get("messages").and_then(Value::as_array) { + if let Some(messages) = body.get("messages") { + let messages = messages + .as_array() + .ok_or_else(|| TranslationError::InvalidType { + path: "$.messages".to_string(), + expected: "array", + })?; let mut generated_id = 0; for (index, message) in messages.iter().enumerate() { let Some(message) = message.as_object() else { diff --git a/crates/switchyard-translation/tests/request_translation.rs b/crates/switchyard-translation/tests/request_translation.rs index e689e5b21..c754994ce 100644 --- a/crates/switchyard-translation/tests/request_translation.rs +++ b/crates/switchyard-translation/tests/request_translation.rs @@ -1782,6 +1782,18 @@ fn malformed_request_fields_are_rejected() { json!({"model": "claude", "max_tokens": "8", "messages": []}), "invalid value at $.max_tokens: expected a non-negative integer", ), + ( + "OpenAI Chat string messages", + WireFormat::OpenAiChat, + json!({"model": "gpt", "messages": "invalid"}), + "expected array at $.messages", + ), + ( + "Anthropic string messages", + WireFormat::AnthropicMessages, + json!({"model": "claude", "max_tokens": 8, "messages": "invalid"}), + "expected array at $.messages", + ), ( "Responses boolean input", WireFormat::OpenAiResponses, diff --git a/docs/operations/soak_test.md b/docs/operations/soak_test.md new file mode 100644 index 000000000..c0ce76cb4 --- /dev/null +++ b/docs/operations/soak_test.md @@ -0,0 +1,342 @@ +# Soak test a release candidate + +A Switchyard soak test sends sustained traffic through a release-candidate +server long enough to expose failures that short tests miss. Run it for 48 +hours before code freeze when a release changes libsy, the Rust server, +routing, streaming, translation, or server lifecycle behavior. + +The standard test sends closed-loop traffic through: + +- OpenAI Chat Completions (`/v1/chat/completions`) +- Anthropic Messages (`/v1/messages`) +- OpenAI Responses (`/v1/responses`) +- streaming and non-streaming responses +- short and long inputs, long outputs, shared prefixes, and mixed request sizes +- growing conversations, large tool catalogs, tool-call bursts, and stage transitions +- deterministic easy/hard classifier mixes + +The runner also checks `/health` and `/metrics` every minute. Every five +minutes, it sends an invalid Chat Completions request, expects HTTP 400, and +then confirms the server is still live. + +The scenario catalog covers these distinct pressure angles: + +| Scenario | Pressure angle | What to review | +|---|---|---| +| `short-interactive` | Short prompts under fixed load, a concurrency knee, or a 10x request-rate burst | HTTP ceiling, TTFT, routing overhead, and the saturation point | +| `long-context` | 8K, 32K, and near-window inputs | TTFT, memory, and context-sensitive route failures | +| `decode-heavy` | 512-token and 1,024-token output limits | ITL, output tokens/second, and stream stability | +| `prefix-reuse` | Matched shared and unique long prefixes | Cache-sensitive TTFT and token accounting | +| `mixed-traffic` | A 70/20/10 short, medium, and long mix | p99 latency and head-of-line effects | +| `growing-conversation` | Eight cumulative conversation turns in one session | Affinity, history cost, and per-turn latency growth | +| `large-tool-catalog` | 16-tool and 64-tool JSON-schema catalogs | Serialization/routing overhead and intact tool forwarding | +| `tool-call-burst` | Eight linked assistant call/tool-result turns | Session continuity and burst handling | +| `stage-transitions` | One growing history across exploration, critical failure, and productive work | Stage-router tier changes and scorer output | +| `classifier-mix` | Deterministic 80/20 then 50/50 easy/hard requests | Target share, classifier calls/errors, and classifier latency | +| `context-overflow` | One target rejects a near-window request | Fallback to another eligible target | +| `failure-pressure` | Bounded 429, 500, malformed verdict, and truncated stream | Retry recovery, explicit terminal errors, and connection health | +| `client-cancellation` | A client timeout during a delayed response | Teardown and recovery of later traffic | + +`standard` includes the core and agentic rows. Run `resilience` separately because expected +failures should not be compared as throughput samples. + +## Prepare the server + +Run the exact commit, build, server config, backend, and model planned for the +release. Do not use a development server in front of a different Switchyard +build. + +Start the standalone server and its libsy algorithms: + +```bash +cargo build --release -p switchyard-server +target/release/switchyard-server --config release-routes.toml \ + > switchyard-soak.log 2>&1 & +SOAK_SERVER_PID=$! +``` + +Wait for `GET http://127.0.0.1:4000/health` to return HTTP 200 with the JSON +body `{"status": "ok"}`; the runner requires both before it starts. Check +`GET http://127.0.0.1:4000/v1/models` and choose the model id that represents +the release workload. Pass that exact id with `--model`. + +Run the server and test from a dedicated host, job scheduler, or terminal +multiplexer that will stay alive for the full test. Confirm that the host will +not suspend or restart and has enough disk space for the server log. + +## Build the soak tester + +The soak tester is a Rust binary. Build it once from the same checkout as the +release, then run it directly: + +```bash +cargo build --release -p switchyard-soak +``` + +## Compare routing algorithm performance + +Use `scripts/benchmark_routing_algorithms.py` to compare routing algorithms under the same load. +The command runs oha and AIPerf sequentially for every model id, then writes `report.md`, +`report.csv`, `report.json`, and `routing-overhead.svg` beside both tools' raw results: + +```bash +python3.12 scripts/benchmark_routing_algorithms.py \ + --base-url http://127.0.0.1:4000 \ + --direct-base-url http://127.0.0.1:8100 \ + --direct-model mock/weak \ + --model noop=switchyard/noop \ + --model passthrough=switchyard/passthrough \ + --model random=switchyard/random \ + --model llm_classifier=switchyard/classifier \ + --model stage_router=switchyard/stage \ + --concurrency 100 \ + --request-count 1000 \ + --scenario-set standard \ + --load-profile fixed \ + --profile-runs 3 \ + --backend-label "release model deployment" +``` + +`--direct-base-url` and `--direct-model` add an AIPerf arm that calls the backend without +Switchyard. The report writes the end-to-end latency and throughput difference for every routed +arm. That difference isolates Switchyard overhead only when the direct and routed arms use the +same backend deployment, model, and response settings. Resilience scenarios have different +failure semantics, so the script does not run or compare the direct arm for them. For an +authenticated backend, pass the name of the key variable, not the key itself, with +`--direct-api-key-env NVIDIA_API_KEY`. + +The Markdown report starts with one overhead row for each route and workload. Positive request or +TTFT latency means the routed request took longer than the direct request. Negative request or +token throughput means the routed path processed less work. The `standard` scenario set includes +all non-resilience request patterns exported by the Rust crate: short and long contexts, long +outputs, shared prefixes, mixed request sizes, growing conversations, large tool catalogs, +tool-call bursts, stage changes, and easy/hard classifier mixes. + +When the run includes a direct backend, the Markdown report embeds `routing-overhead.svg` before +the overhead table. The two annotated heatmaps show TTFT and output-token-throughput changes for +every route and workload. Each cell shows the absolute change in milliseconds or tokens per second +and the percent change. Red cells are worse than the direct backend, and blue cells are better. +The script uses only the Python standard library to create the SVG, so the same benchmark command +reproduces the plot without a plotting package. + +The Rust crate exports one AIPerf `inputs-json` file per scenario. oha reuses the first exported +`short-interactive` payload as a non-streaming fixed body and reports the raw HTTP request rate and +latency ceiling. AIPerf replays every selected streaming session and reports request latency, time +to first token (TTFT), inter-token latency (ITL), request throughput, output-token throughput, and +multi-run confidence intervals. The command saves `/v1/stats` before and after each scenario/load +cell, so the routing calls and errors total all independent repetitions in that cell. The report +also records selected-target shares, classifier calls/errors and latency, and mean routing +overhead. It runs jobs sequentially because simultaneous tools would compete for the same server +capacity. + +Use the load schedules after the fixed scenario comparison establishes a baseline: + +```bash +python3.12 scripts/benchmark_routing_algorithms.py \ + --base-url http://127.0.0.1:4000 \ + --model random=switchyard/random \ + --model classifier=switchyard/classifier \ + --scenario short-interactive \ + --load-profile concurrency-knee \ + --load-profile traffic-burst \ + --concurrency 128 \ + --request-rate 20 +``` + +The concurrency knee uses bounded steps up to `--concurrency`. The traffic burst holds the base +request rate, raises it to 10 times that rate for five seconds, then returns to the base. Pass +`--load-profile all` to run fixed, knee, and burst schedules. These schedules apply to the short +baseline; they are not separate request scenarios. + +To isolate routing overhead, configure every route to use the same target deployment and keep the +scenario manifest, concurrency, request count, and profile-run count fixed. Set `--tokenizer` to +the real model tokenizer when exact token counts matter. AIPerf uses deterministic sessions and a +fixed seed so each route receives the same requests. + +This comparison command does not start a backend or Switchyard. Point it at a running Switchyard +server backed by real models to measure end-to-end TTFT and token throughput with real tokenization +and generated tokens. Real-model runs cost tokens and include provider queuing and model variance, +so use a dedicated deployment and repeat the run before treating a small difference as an +algorithm effect. Use the local scenario backend first for deterministic routing correctness and +overhead, then rerun the same manifest against real models for capacity claims. + +## Check the local server and load tools + +The local test uses a request-aware Axum backend from the soak crate. Build the server, soak tester, +and scenario backend from the commit under test: + +```bash +cargo build --release -p switchyard-server -p switchyard-soak \ + --bins --example switchyard-soak-mock +``` + +Install [oha](https://github.com/hatoo/oha) and +[NVIDIA AIPerf](https://docs.nvidia.com/aiperf/reference/command-line-options): + +```bash +cargo install oha +uv tool install --python 3.12 'aiperf==0.11.0' +``` + +The benchmark checks the AIPerf version before it creates an output directory. It runs each +repetition in a separate AIPerf process and calculates confidence intervals from those independent +runs. Each process starts one required record processor before profiling. The benchmark gives each +process a deadline based on its request schedule, then stops the whole process group if that deadline +expires. This prevents a startup failure from leaving a benchmark stuck at zero processed records. + +Then run the local test from the repository root: + +```bash +python3.12 scripts/run_local_soak_test.py \ + --duration 10s \ + --concurrency 4 \ + --request-count 100 +``` + +The local backend waits 40 ms before the first token and 1 ms between output tokens by default. +It streams the output limit from each Rust scenario, so decode-heavy requests produce 512 or 1,024 +tokens and AIPerf can measure TTFT, ITL, and token throughput separately. Use +`--mock-latency-ms` and `--mock-token-latency-ms` to change that deterministic timing model. + +`--duration` controls how long the Rust soak tester runs. `--request-count` controls how many +measured requests each load tool sends to each algorithm. `--help` explains every flag. Set +`OHA_BIN`, `AIPERF_BIN`, +`SWITCHYARD_SERVER_BIN`, `SWITCHYARD_SOAK_BIN`, or `SWITCHYARD_SOAK_MOCK_BIN` when a command is not +on `PATH` or not under `target/release`. + +The script checks all five commands before starting a process. A missing command prints a warning, +the build or install command, and the matching environment variable. Cargo builds the scenario +backend, but it does not install oha or AIPerf during a normal build. + +The script gives each tool one job: + +| Tool | Job in the local test | +|---|---| +| Scenario backend | Returns local OpenAI-compatible responses, valid easy/hard classifier verdicts, and bounded failures with no provider cost. | +| oha | Measures the non-streaming `short-interactive` HTTP baseline through every route. | +| Python route checks | Sends one ordinary Chat Completions request through each configured route before load starts. | +| AIPerf | Replays Rust-exported streaming sessions through every route and records LLM, token, response-time, and confidence results. | +| Combined report | Joins scenario, load, oha, AIPerf, and routing-counter metrics in Markdown, CSV, JSON, and an overhead plot. It keeps resilience rows separate from throughput rows. | +| `switchyard-soak` | Runs the standard scenario set while checking public API variants, server health, metrics, process use, and required results. | + +`scripts/local_soak_test.toml` exercises `noop`, `random`, `passthrough`, `llm_classifier`, and +`stage_router`. It uses the accepted maximum retry count (10), a zero-weight random target, and the +upper classifier and stage thresholds (1.0). Classifier affinity is disabled so every measured +request includes the classifier call. The scenario backend returns `p_solve=1.0` for easy markers +and `p_solve=0.1` for hard markers; with the configured threshold, those requests select the weak +and strong targets respectively. The config is validated with +`switchyard-server --dry-run` before either service starts. + +Run resilience cases separately so expected transport failures do not contaminate throughput +comparisons: + +```bash +python3.12 scripts/benchmark_routing_algorithms.py \ + --base-url http://127.0.0.1:4000 \ + --model classifier=switchyard/classifier \ + --scenario-set resilience \ + --load-profile fixed \ + --profile-runs 1 +``` + +`context-overflow` checks target fallback, `failure-pressure` injects bounded 429, 500, malformed +classifier, and truncated-stream cases, and `client-cancellation` uses a one-second client timeout +against a delayed response. Their expected error-rate ranges appear in the report's Resilience +section, and the command fails after writing the report when any row misses its range. The local +runner calls the scenario backend's `/reset` endpoint before each AIPerf cell so every algorithm +receives the same transient-failure sequence. When you run the comparison command directly against +that backend, pass `--scenario-backend-reset-url http://127.0.0.1:8100/reset` to preserve the same +comparison. + +The runner does not add limits for target count or recent-turn history because the server does not +limit them. Rust tests cover the real limits: 4,096 saved route assignments, 100,000 response-time +samples, and 10,000 error records. Server tests cover invalid thresholds, bad classifier responses, +missing stage-router request IDs, target errors, and retry count 11, which is one above the maximum. + +## Run the 48-hour test + +Choose concurrency from the release capacity plan. Increase it in short +test runs until you find the highest expected steady load that remains below +the backend's rate limit. Use that load for the 48-hour run. An overload test +that spends most of its time throttled does not measure release stability. + +```bash +./target/release/switchyard-soak \ + --base-url http://127.0.0.1:4000 \ + --model RELEASE_MODEL_ID \ + --duration 48h \ + --concurrency 16 \ + --server-pid "$SOAK_SERVER_PID" \ + --max-rss-growth-mib 512 +``` + +The runner keeps 16 requests in flight until the test ends. This can generate +large usage charges against a metered backend. Use a dedicated test deployment, +estimate the request volume first with a short run, and get approval for any +paid-provider cost. + +Use a five-minute run to confirm the route and result files: + +```bash +./target/release/switchyard-soak \ + --base-url http://127.0.0.1:4000 \ + --model RELEASE_MODEL_ID \ + --duration 5m \ + --concurrency 4 \ + --report-interval 10 +``` + +If the Switchyard endpoint requires a bearer token, pass the environment +variable name instead of putting the token on the command line: + +```bash +export SWITCHYARD_SOAK_TOKEN="..." +./target/release/switchyard-soak \ + --api-key-env SWITCHYARD_SOAK_TOKEN \ + --model RELEASE_MODEL_ID +``` + +## Pass criteria + +The command exits with status 0 only when: + +- the requested duration completes; +- at least one inference request completes; +- the inference error rate stays at or below `--max-error-rate` (default `0`, + which means no inference request may fail); +- every periodic liveness check passes; +- every `/metrics` read returns both Switchyard request counters; +- every requested process sample returns RSS and CPU data; +- every invalid-request recovery check passes; +- the server request counter never resets; and +- RSS growth stays within `--max-rss-growth-mib` when that limit is set. + +If the release plan permits transient failures from a remote provider, set an +explicit error budget with `--max-error-rate`. Record the reason for that +exception in the release record. + +The RSS limit is deployment-specific. Set it from an approved baseline for the +same model, concurrency, and worker count. Omit `--server-pid` and +`--max-rss-growth-mib` when the server runs on another host, then collect +memory and restart data from that host's monitoring system. + +## Review the results + +Each run creates a timestamped directory under `soak-results/`: + +- `config.json` records the non-secret test inputs. +- `intervals.csv` records request rate, errors, latency percentiles, health, + Switchyard counters, RSS, and CPU once per reporting interval. `cpu_percent` + is the `ps` lifetime-average CPU for the process, not the interval's usage, so + read it as a long-run average rather than a spike detector. +- `errors.jsonl` records up to 10,000 request and canary failures. +- `summary.json` records the final pass result and any failed gates. + +Tail the run log to monitor progress, cumulative error rate, health, RSS, and the +`OK`, `DEGRADED`, or `STALLED` interval status while the test runs. + +Before approving the release, check `intervals.csv` for late failures, falling +throughput, increasing p95 or p99 latency, and steady RSS growth. Compare the +first and last several hours, not only the run-wide averages. Attach +`summary.json`, the interval chart, the Switchyard log, the tested commit, and +the server config to the release record. diff --git a/mkdocs.yml b/mkdocs.yml index fcf40f715..d934b61ef 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -32,6 +32,7 @@ nav: - Advisor-Gate Routing: routing_algorithms/advisor_gate_routing.md - Operations: - Context-Window Handling: operations/context_window.md + - Soak Testing: operations/soak_test.md - Reference: - CLI Reference: cli_reference.md - TOML Schema: reference/toml_schema.md diff --git a/scripts/aiperf_runner.py b/scripts/aiperf_runner.py new file mode 100644 index 000000000..f83165df1 --- /dev/null +++ b/scripts/aiperf_runner.py @@ -0,0 +1,220 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Run the supported AIPerf release without allowing its startup race to hang.""" + +import json +import math +import os +import signal +import statistics +import subprocess +import time +from collections.abc import Sequence +from pathlib import Path + +SUPPORTED_AIPERF_VERSION = "0.11.0" +PROCESS_GROUP_GRACE_SECONDS = 5 +_T_CRITICAL_95 = { + 2: 12.706, + 3: 4.303, + 4: 3.182, + 5: 2.776, + 6: 2.571, + 7: 2.447, + 8: 2.365, + 9: 2.306, + 10: 2.262, +} +_REQUIRED_STATISTICS = ( + ("error_request_count", "avg"), + ("request_count", "avg"), + ("request_throughput", "avg"), +) + + +def validate_aiperf_version(binary: str) -> None: + """Reject AIPerf versions whose CLI or startup behavior is not covered.""" + try: + result = subprocess.run( + [binary, "--version"], + capture_output=True, + text=True, + check=False, + timeout=10, + ) + except (OSError, subprocess.TimeoutExpired) as error: + raise RuntimeError(f"could not read the AIPerf version from {binary}: {error}") from error + version = result.stdout.strip() + if result.returncode != 0 or version != SUPPORTED_AIPERF_VERSION: + found = version or result.stderr.strip() or "unknown" + raise RuntimeError( + f"AIPerf {found} is unsupported; install {SUPPORTED_AIPERF_VERSION} with: " + f"uv tool install --python 3.12 'aiperf=={SUPPORTED_AIPERF_VERSION}'" + ) + + +def _process_group_exists(process_group: int) -> bool: + try: + os.killpg(process_group, 0) + except ProcessLookupError: + return False + except PermissionError: + # A completed leader's process-group id can be reused by a process we do not own. + return False + return True + + +def _stop_process_group(process: subprocess.Popen[bytes]) -> None: + """Stop AIPerf and every worker process it started.""" + process_group = process.pid + try: + os.killpg(process_group, signal.SIGTERM) + except ProcessLookupError: + pass + except PermissionError as error: + if process.poll() is None: + raise RuntimeError( + f"could not stop running AIPerf process group {process_group}" + ) from error + return + deadline = time.monotonic() + PROCESS_GROUP_GRACE_SECONDS + while _process_group_exists(process_group) and time.monotonic() < deadline: + process.poll() + time.sleep(0.05) + if _process_group_exists(process_group): + try: + os.killpg(process_group, signal.SIGKILL) + except ProcessLookupError: + pass + except PermissionError as error: + if process.poll() is None: + raise RuntimeError( + f"could not kill running AIPerf process group {process_group}" + ) from error + try: + process.wait(timeout=PROCESS_GROUP_GRACE_SECONDS) + except subprocess.TimeoutExpired as error: + raise RuntimeError(f"could not reap AIPerf process {process.pid}") from error + + +def run_profile( + command: Sequence[str], + log_path: Path, + artifact_dir: Path, + timeout_seconds: int, +) -> Path: + """Run one bounded AIPerf process and return its verified export.""" + artifact_dir.parent.mkdir(parents=True, exist_ok=True) + with log_path.open("wb") as log: + try: + process = subprocess.Popen( + [*command, "--artifact-dir", str(artifact_dir)], + stdout=log, + stderr=subprocess.STDOUT, + start_new_session=True, + ) + except OSError as error: + raise RuntimeError(f"could not start AIPerf: {error}") from error + stopped = False + try: + try: + returncode = process.wait(timeout=timeout_seconds) + except subprocess.TimeoutExpired as error: + timeout_message = f"AIPerf exceeded its {timeout_seconds}-second run limit" + log.write(f"\n{timeout_message}.\n".encode()) + log.flush() + _stop_process_group(process) + stopped = True + raise RuntimeError(f"{timeout_message}; see {log_path}") from error + finally: + if not stopped and (process.poll() is None or _process_group_exists(process.pid)): + _stop_process_group(process) + if returncode != 0: + raise RuntimeError(f"AIPerf failed with status {returncode}; see {log_path}") + export_path = artifact_dir / "profile_export_aiperf.json" + if not export_path.is_file(): + raise RuntimeError(f"AIPerf did not write {export_path.name}; see {log_path}") + return export_path + + +def _summary(values: Sequence[float], unit: str) -> dict[str, float | str | None]: + """Return the 95% confidence summary used by the benchmark report.""" + count = len(values) + mean = statistics.fmean(values) + standard_deviation = statistics.stdev(values) if count > 1 else 0.0 + standard_error = standard_deviation / math.sqrt(count) + critical = _T_CRITICAL_95.get(count) + margin = critical * standard_error if critical is not None else 0.0 + return { + "mean": mean, + "std": standard_deviation, + "min": min(values), + "max": max(values), + "cv": standard_deviation / mean if mean else 0.0, + "se": standard_error, + "ci_low": mean - margin, + "ci_high": mean + margin, + "t_critical": critical, + "unit": unit, + } + + +def aggregate_exports(exports: Sequence[Path], output_path: Path) -> Path: + """Combine independent AIPerf exports into the aggregate schema consumed by the report.""" + if not 2 <= len(exports) <= 10: + raise RuntimeError("AIPerf aggregation requires between 2 and 10 exports") + collected: dict[tuple[str, str, str], list[float]] = {} + metric_units: dict[tuple[str, str], str] = {} + for path in exports: + try: + document = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + raise RuntimeError(f"could not read AIPerf export {path}: {error}") from error + if not isinstance(document, dict): + raise RuntimeError(f"AIPerf export is not a JSON object: {path}") + version = document.get("aiperf_version") + if version != SUPPORTED_AIPERF_VERSION: + raise RuntimeError( + f"AIPerf export {path} has version {version!r}; expected {SUPPORTED_AIPERF_VERSION}" + ) + error_count = document.get("error_request_count") + clean_error_summary = document.get("error_summary") == [] + if error_count is None and ("error_request_count" in document or clean_error_summary): + document["error_request_count"] = {"unit": "requests", "avg": 0.0} + for metric_name, statistic in _REQUIRED_STATISTICS: + metric = document.get(metric_name) + value = metric.get(statistic) if isinstance(metric, dict) else None + unit = metric.get("unit") if isinstance(metric, dict) else None + if ( + not isinstance(value, int | float) + or isinstance(value, bool) + or not isinstance(unit, str) + ): + raise RuntimeError(f"AIPerf export {path} has no numeric {metric_name}.{statistic}") + for metric_name, metric in document.items(): + if not isinstance(metric, dict) or not isinstance(metric.get("unit"), str): + continue + unit = metric["unit"] + for statistic, value in metric.items(): + if statistic == "unit" or isinstance(value, bool): + continue + if isinstance(value, int | float): + previous_unit = metric_units.setdefault((metric_name, statistic), unit) + if unit != previous_unit: + raise RuntimeError( + "AIPerf exports disagree on the unit for " + f"{metric_name}.{statistic}: {previous_unit!r} and {unit!r}" + ) + collected.setdefault((metric_name, statistic, unit), []).append(float(value)) + metrics = { + f"{metric}_{statistic}": _summary(values, unit) + for (metric, statistic, unit), values in collected.items() + if len(values) == len(exports) + } + output_path.parent.mkdir(parents=True, exist_ok=False) + output_path.write_text( + f"{json.dumps({'aiperf_version': SUPPORTED_AIPERF_VERSION, 'metrics': metrics}, indent=2)}\n", + encoding="utf-8", + ) + return output_path diff --git a/scripts/benchmark_routing_algorithms.py b/scripts/benchmark_routing_algorithms.py new file mode 100755 index 000000000..9a0a8baa7 --- /dev/null +++ b/scripts/benchmark_routing_algorithms.py @@ -0,0 +1,1523 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Compare Switchyard routes with Rust-owned scenarios, oha, and NVIDIA AIPerf.""" + +import argparse +import csv +import json +import math +import os +import shutil +import subprocess +import sys +import urllib.error +import urllib.request +from collections.abc import Sequence +from dataclasses import asdict, dataclass, replace +from datetime import datetime, timezone +from pathlib import Path + +if __package__: + from .aiperf_runner import aggregate_exports, run_profile, validate_aiperf_version + from .routing_overhead_plot import OverheadPlotRow, write_overhead_plot +else: + from aiperf_runner import aggregate_exports, run_profile, validate_aiperf_version + from routing_overhead_plot import OverheadPlotRow, write_overhead_plot + +MAX_MATERIALIZED_REQUESTS = 1_000_000 +MAX_MATERIALIZED_BYTES = 256 * 1024 * 1024 +AIPERF_LIFECYCLE_ALLOWANCE_SECONDS = 120 + + +@dataclass(frozen=True) +class RequiredBinary: + """One command the benchmark must find before it starts.""" + + label: str + env_var: str + value: str + setup: str + + +@dataclass(frozen=True) +class DirectBaseline: + """A backend endpoint that AIPerf calls without going through Switchyard.""" + + base_url: str + model: str + api_key_env: str | None = None + + +@dataclass(frozen=True) +class BenchmarkConfig: + """Inputs shared by every algorithm comparison run.""" + + base_url: str + models: tuple[tuple[str, str], ...] + concurrency: int + request_count: int + backend_label: str + output_dir: Path + oha_bin: str + aiperf_bin: str + soak_bin: str + tokenizer: str = "builtin" + scenario_set: str = "standard" + scenarios: tuple[str, ...] = () + load_profiles: tuple[str, ...] = ("fixed",) + profile_runs: int = 3 + prompt_bytes: int = 1024 + max_output_tokens: int = 32 + context_window_tokens: int = 32_768 + request_rate: float | None = None + scenario_backend_reset_url: str | None = None + direct_baseline: DirectBaseline | None = None + + +@dataclass(frozen=True) +class BenchmarkArm: + """One endpoint and model pair included in the comparison.""" + + label: str + model: str + base_url: str + bypasses_switchyard: bool + api_key_env: str | None = None + + +@dataclass(frozen=True) +class ScenarioDefinition: + """One validated entry exported by switchyard-soak.""" + + id: str + group: str + description: str + expected: str + expected_error_rate_min: float + expected_error_rate_max: float + input_file: Path + load_profiles: tuple[dict[str, object], ...] + + +@dataclass(frozen=True) +class RoutingDelta: + """Routing counters attributable to one sequential AIPerf run.""" + + model_calls: str + model_share: str + model_errors: str + classifier_calls: int + classifier_errors: int + classifier_latency_avg_ms: float | None + routing_overhead_avg_ms: float | None + + +@dataclass(frozen=True) +class BenchmarkResult: + """Selected load-tool and routing metrics for one comparison row.""" + + algorithm: str + model: str + scenario: str + scenario_description: str + scenario_group: str + load_profile: str + expected_behavior: str + expected_error_rate_min: float + expected_error_rate_max: float + expectation_met: bool + oha_requests_per_second: float | None + oha_latency_p50_ms: float | None + oha_latency_p99_ms: float | None + aiperf_requests_per_second: float | None + aiperf_request_latency_p50_ms: float | None + aiperf_ttft_p50_ms: float | None + aiperf_ttft_p99_ms: float | None + aiperf_itl_p50_ms: float | None + aiperf_output_tokens_per_second: float | None + aiperf_output_tokens_per_second_per_user_p50: float | None + aiperf_error_requests: float + aiperf_error_rate: float + aiperf_request_throughput_cv: float | None + aiperf_request_throughput_ci_low: float | None + aiperf_request_throughput_ci_high: float | None + selected_model_calls: str + selected_model_share: str + selected_model_errors: str + classifier_calls: int + classifier_errors: int + classifier_latency_avg_ms: float | None + routing_overhead_avg_ms: float | None + bypasses_switchyard: bool + aiperf_request_throughput_delta_pct: float | None = None + aiperf_request_latency_p50_delta_ms: float | None = None + aiperf_ttft_p50_delta_ms: float | None = None + aiperf_ttft_p50_delta_pct: float | None = None + aiperf_ttft_p99_delta_ms: float | None = None + aiperf_output_tokens_per_second_delta: float | None = None + aiperf_output_tokens_per_second_delta_pct: float | None = None + + +def positive_int(value: str) -> int: + """Parse a command-line integer that must be greater than zero.""" + parsed = int(value) + if parsed <= 0: + raise argparse.ArgumentTypeError("must be greater than zero") + return parsed + + +def positive_float(value: str) -> float: + """Parse a finite command-line number that must be greater than zero.""" + parsed = float(value) + if parsed <= 0 or not math.isfinite(parsed): + raise argparse.ArgumentTypeError("must be a finite number greater than zero") + return parsed + + +def model_spec(value: str) -> tuple[str, str]: + """Parse LABEL=MODEL without coupling the report label to a filesystem name.""" + label, separator, model = value.partition("=") + if not separator or not label.strip() or not model.strip(): + raise argparse.ArgumentTypeError("must use LABEL=MODEL") + return label.strip(), model.strip() + + +def parser() -> argparse.ArgumentParser: + """Build the command-line interface.""" + command = argparse.ArgumentParser( + description=( + "Export Rust-owned request scenarios, run oha for the short baseline and AIPerf " + "for every selected scenario, then write one routing performance report." + ), + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + command.add_argument("--base-url", required=True, help="URL of a running Switchyard server") + command.add_argument( + "--direct-base-url", + help="backend URL for an AIPerf baseline that bypasses Switchyard", + ) + command.add_argument( + "--direct-model", + help="backend model id for the direct baseline; requires --direct-base-url", + ) + command.add_argument( + "--direct-api-key-env", + help="environment variable containing the direct backend API key", + ) + command.add_argument( + "--model", + action="append", + required=True, + type=model_spec, + dest="models", + metavar="LABEL=MODEL", + help="report label and exact model id; repeat for every route to compare", + ) + command.add_argument("--concurrency", type=positive_int, default=4) + command.add_argument("--request-count", type=positive_int, default=100) + command.add_argument("--tokenizer", default="builtin") + command.add_argument( + "--scenario-set", + choices=("core", "agentic", "resilience", "standard", "all"), + default="standard", + ) + command.add_argument( + "--scenario", + action="append", + default=[], + help="exact Rust scenario id; repeat to preserve an explicit order", + ) + command.add_argument( + "--load-profile", + action="append", + choices=("fixed", "concurrency-knee", "traffic-burst", "all"), + default=[], + help="load schedule to run; repeat as needed", + ) + command.add_argument( + "--profile-runs", + type=positive_int, + default=3, + help="AIPerf repetitions used for confidence intervals; maximum 10", + ) + command.add_argument( + "--request-rate", + type=positive_float, + help="base requests/second for traffic-burst; defaults to concurrency", + ) + command.add_argument("--prompt-bytes", type=positive_int, default=1024) + command.add_argument("--max-output-tokens", type=positive_int, default=32) + command.add_argument("--context-window-tokens", type=positive_int, default=32_768) + command.add_argument("--backend-label", default="unspecified backend") + command.add_argument( + "--scenario-backend-reset-url", + help="optional local scenario-backend endpoint reset before each AIPerf cell", + ) + command.add_argument("--output-dir", type=Path) + command.add_argument("--oha-bin", default=os.environ.get("OHA_BIN", "oha")) + command.add_argument("--aiperf-bin", default=os.environ.get("AIPERF_BIN", "aiperf")) + command.add_argument( + "--soak-bin", + default=os.environ.get("SWITCHYARD_SOAK_BIN", "target/release/switchyard-soak"), + ) + return command + + +def find_binary(value: str) -> str | None: + """Return an executable path, or None when the command cannot run.""" + if os.sep in value: + path = Path(value).expanduser().resolve() + if path.is_file() and os.access(path, os.X_OK): + return str(path) + else: + found = shutil.which(value) + if found is not None: + return found + return None + + +def resolve_binaries(requirements: Sequence[RequiredBinary]) -> dict[str, str]: + """Resolve every command and print all missing-command setup instructions.""" + resolved = {} + missing = [] + for requirement in requirements: + path = find_binary(requirement.value) + if path is None: + missing.append(requirement) + else: + resolved[requirement.label] = path + for requirement in missing: + print( + f"warning: {requirement.label} executable not found: {requirement.value}", + file=sys.stderr, + ) + print(f" {requirement.setup}", file=sys.stderr) + print( + f" Or set {requirement.env_var}=/path/to/{Path(requirement.value).name}", + file=sys.stderr, + ) + if missing: + raise RuntimeError(f"install or build the {len(missing)} missing command(s), then rerun") + return resolved + + +def run_checked(name: str, command: Sequence[str], log_path: Path) -> None: + """Run one finite tool and keep its output in a named log file.""" + print(f"Running {name}; log: {log_path}") + with log_path.open("w", encoding="utf-8") as log: + result = subprocess.run( + list(command), + stdout=log, + stderr=subprocess.STDOUT, + text=True, + check=False, + ) + if result.returncode != 0: + raise RuntimeError(f"{name} failed with status {result.returncode}; see {log_path}") + + +def read_json_object(path: Path) -> dict[str, object]: + """Read a JSON object and name the invalid result file in any error.""" + try: + document = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + raise RuntimeError(f"could not read JSON result {path}: {error}") from error + if not isinstance(document, dict): + raise RuntimeError(f"expected a JSON object in {path}") + return document + + +def capture_stats(base_url: str, path: Path) -> dict[str, object]: + """Capture Switchyard's routing counters before or after one isolated run.""" + try: + with urllib.request.urlopen(f"{base_url.rstrip('/')}/v1/stats", timeout=10) as response: + document = json.loads(response.read()) + except (OSError, urllib.error.URLError, json.JSONDecodeError) as error: + raise RuntimeError(f"could not read Switchyard stats: {error}") from error + if not isinstance(document, dict): + raise RuntimeError("Switchyard /v1/stats did not return a JSON object") + path.write_text(f"{json.dumps(document, indent=2)}\n", encoding="utf-8") + return document + + +def reset_scenario_backend(url: str | None) -> None: + """Reset local failure injection so every algorithm receives the same attempts.""" + if url is None: + return + request = urllib.request.Request(url, data=b"", method="POST") + try: + with urllib.request.urlopen(request, timeout=10) as response: + document = json.loads(response.read()) + except (OSError, urllib.error.URLError, json.JSONDecodeError) as error: + raise RuntimeError(f"could not reset the scenario backend: {error}") from error + status = document.get("status") if isinstance(document, dict) else None + if response.status != 200 or status != "reset": + raise RuntimeError(f"scenario backend reset returned an invalid response from {url}") + + +def _nested_number(document: dict[str, object], section: str, field: str) -> float: + section_value = document.get(section) + value = section_value.get(field) if isinstance(section_value, dict) else None + return float(value) if isinstance(value, int | float) else 0.0 + + +def _histogram_delta( + before: dict[str, object], after: dict[str, object], histogram: str +) -> float | None: + count = _nested_number(after, histogram, "count") - _nested_number(before, histogram, "count") + total = _nested_number(after, histogram, "total_ms") - _nested_number( + before, histogram, "total_ms" + ) + return total / count if count > 0 else None + + +def stats_delta(before: dict[str, object], after: dict[str, object]) -> RoutingDelta: + """Return routing and target-counter changes for one run.""" + before_value = before.get("models") + after_value = after.get("models") + before_models: dict[str, object] = before_value if isinstance(before_value, dict) else {} + after_models: dict[str, object] = after_value if isinstance(after_value, dict) else {} + classifier_before = before.get("classifier") + classifier_after = after.get("classifier") + before_classifier_models = ( + classifier_before.get("models") if isinstance(classifier_before, dict) else None + ) + after_classifier_models = ( + classifier_after.get("models") if isinstance(classifier_after, dict) else None + ) + calls = {} + errors = {} + for model, current in after_models.items(): + if not isinstance(model, str) or not isinstance(current, dict): + continue + previous = before_models.get(model) + previous_calls = previous.get("calls", 0) if isinstance(previous, dict) else 0 + classifier_current = ( + after_classifier_models.get(model) + if isinstance(after_classifier_models, dict) + else None + ) + classifier_previous = ( + before_classifier_models.get(model) + if isinstance(before_classifier_models, dict) + else None + ) + classifier_call_delta = float( + classifier_current.get("calls", 0) if isinstance(classifier_current, dict) else 0 + ) - float( + classifier_previous.get("calls", 0) if isinstance(classifier_previous, dict) else 0 + ) + call_delta = int( + float(current.get("calls", 0)) - float(previous_calls) - classifier_call_delta + ) + if call_delta > 0: + calls[model] = call_delta + previous_errors = previous.get("errors", 0) if isinstance(previous, dict) else 0 + classifier_error_delta = float( + classifier_current.get("errors", 0) if isinstance(classifier_current, dict) else 0 + ) - float( + classifier_previous.get("errors", 0) if isinstance(classifier_previous, dict) else 0 + ) + error_delta = int( + float(current.get("errors", 0)) - float(previous_errors) - classifier_error_delta + ) + if error_delta > 0: + errors[model] = error_delta + + classifier_count = classifier_total = 0.0 + if isinstance(after_classifier_models, dict): + for model, current in after_classifier_models.items(): + if not isinstance(model, str) or not isinstance(current, dict): + continue + previous = ( + before_classifier_models.get(model) + if isinstance(before_classifier_models, dict) + else None + ) + before_record = previous if isinstance(previous, dict) else {} + classifier_count += _nested_number(current, "model_call_latency", "count") + classifier_count -= _nested_number(before_record, "model_call_latency", "count") + classifier_total += _nested_number(current, "model_call_latency", "total_ms") + classifier_total -= _nested_number(before_record, "model_call_latency", "total_ms") + + total_calls = sum(calls.values()) + shares = {model: round(call_count / total_calls, 4) for model, call_count in calls.items()} + return RoutingDelta( + model_calls=json.dumps(calls, sort_keys=True, separators=(",", ":")), + model_share=json.dumps(shares, sort_keys=True, separators=(",", ":")), + model_errors=json.dumps(errors, sort_keys=True, separators=(",", ":")), + classifier_calls=int( + _nested_number(after, "classifier", "total_requests") + - _nested_number(before, "classifier", "total_requests") + ), + classifier_errors=int( + _nested_number(after, "classifier", "total_errors") + - _nested_number(before, "classifier", "total_errors") + ), + classifier_latency_avg_ms=( + classifier_total / classifier_count if classifier_count > 0 else None + ), + routing_overhead_avg_ms=_histogram_delta(before, after, "routing_overhead"), + ) + + +def validate_load_profile(profile: dict[str, object], scenario_id: str) -> None: + """Reject malformed or unbounded schedules before invoking a load generator.""" + profile_id = profile.get("id") + kind = profile.get("kind") + if (profile_id, kind) == ("fixed", "fixed"): + return + if (profile_id, kind) == ("concurrency-knee", "concurrency_knee"): + steps = profile.get("concurrency_steps") + if ( + not isinstance(steps, list) + or not steps + or not all(isinstance(step, int) and 1 <= step <= 10_000 for step in steps) + ): + raise RuntimeError(f"invalid concurrency knee for scenario {scenario_id}") + return + if (profile_id, kind) == ("traffic-burst", "traffic_burst"): + duration = profile.get("duration_seconds") + points = profile.get("points") + if not isinstance(duration, int) or not 1 <= duration <= 3_600: + raise RuntimeError(f"invalid traffic-burst duration for scenario {scenario_id}") + if not isinstance(points, list) or not 2 <= len(points) <= 64: + raise RuntimeError(f"invalid traffic-burst points for scenario {scenario_id}") + previous = -1.0 + for point in points: + time_s = point.get("time_s") if isinstance(point, dict) else None + multiplier = point.get("rate_multiplier") if isinstance(point, dict) else None + if ( + not isinstance(time_s, int | float) + or not isinstance(multiplier, int | float) + or not math.isfinite(float(time_s)) + or not math.isfinite(float(multiplier)) + or float(time_s) <= previous + or float(time_s) > duration + or float(multiplier) <= 0 + ): + raise RuntimeError(f"invalid traffic-burst point for scenario {scenario_id}") + previous = float(time_s) + return + raise RuntimeError(f"unknown load profile for scenario {scenario_id}: {profile_id!r}") + + +def export_scenarios(config: BenchmarkConfig, index: int, model: str) -> list[ScenarioDefinition]: + """Ask the Rust crate for the only authoritative scenario manifest.""" + output_dir = config.output_dir / "scenario-definitions" / f"algorithm-{index:02d}" + command = [ + config.soak_bin, + "--model", + model, + "--scenario-set", + config.scenario_set, + "--prompt-bytes", + str(config.prompt_bytes), + "--max-output-tokens", + str(config.max_output_tokens), + "--context-window-tokens", + str(config.context_window_tokens), + "--export-scenarios", + str(output_dir), + ] + for scenario in config.scenarios: + command.extend(("--scenario", scenario)) + run_checked( + "scenario export", + command, + config.output_dir / f"scenario-export-{index:02d}.log", + ) + document = read_json_object(output_dir / "manifest.json") + if document.get("schema_version") != 1 or document.get("model") != model: + raise RuntimeError(f"unsupported scenario manifest in {output_dir}") + raw_scenarios = document.get("scenarios") + if not isinstance(raw_scenarios, list) or not raw_scenarios: + raise RuntimeError(f"scenario manifest has no scenarios: {output_dir}") + definitions = [] + seen = set() + root = output_dir.resolve() + for entry in raw_scenarios: + if not isinstance(entry, dict): + raise RuntimeError(f"invalid scenario entry in {output_dir}") + scenario_id = entry.get("id") + group = entry.get("group") + description = entry.get("description") + expected = entry.get("expected") + expected_error_rate = entry.get("expected_error_rate") + min_error_rate = ( + expected_error_rate.get("min_rate") if isinstance(expected_error_rate, dict) else None + ) + max_error_rate = ( + expected_error_rate.get("max_rate") if isinstance(expected_error_rate, dict) else None + ) + input_name = entry.get("input_file") + profiles = entry.get("load_profiles") + if ( + not isinstance(scenario_id, str) + or not scenario_id + or scenario_id in seen + or group not in {"core", "agentic", "resilience"} + or not isinstance(description, str) + or not description + or not isinstance(expected, str) + or not expected + or not isinstance(min_error_rate, int | float) + or not isinstance(max_error_rate, int | float) + or not 0 <= float(min_error_rate) <= float(max_error_rate) <= 1 + or not isinstance(input_name, str) + or not isinstance(profiles, list) + ): + raise RuntimeError(f"invalid scenario entry in {output_dir}") + input_file = (output_dir / input_name).resolve() + if not input_file.is_relative_to(root) or not input_file.is_file(): + raise RuntimeError(f"scenario input escapes or is missing from {output_dir}") + if not all( + isinstance(profile, dict) and isinstance(profile.get("id"), str) for profile in profiles + ): + raise RuntimeError(f"invalid load profile for scenario {scenario_id}") + for profile in profiles: + validate_load_profile(profile, scenario_id) + seen.add(scenario_id) + definitions.append( + ScenarioDefinition( + id=scenario_id, + group=group, + description=description, + expected=expected, + expected_error_rate_min=float(min_error_rate), + expected_error_rate_max=float(max_error_rate), + input_file=input_file, + load_profiles=tuple(profiles), + ) + ) + return definitions + + +def selected_profiles( + config: BenchmarkConfig, scenario: ScenarioDefinition +) -> list[dict[str, object]]: + """Select only schedules supported by this request shape.""" + requested = set(config.load_profiles) + if "all" in requested: + return list(scenario.load_profiles) + return [profile for profile in scenario.load_profiles if profile.get("id") in requested] + + +def _metric(document: dict[str, object], name: str, statistic: str) -> tuple[float | None, object]: + group = document.get(name) + if isinstance(group, dict): + value = group.get(statistic) + return (float(value) if isinstance(value, int | float) else None, group.get("unit")) + metrics = document.get("metrics") + aggregate = metrics.get(f"{name}_{statistic}") if isinstance(metrics, dict) else None + if isinstance(aggregate, dict): + value = aggregate.get("mean") + return ( + float(value) if isinstance(value, int | float) else None, + aggregate.get("unit"), + ) + return None, None + + +def _latency(document: dict[str, object], metric: str, statistic: str, path: Path) -> float | None: + value, unit = _metric(document, metric, statistic) + if value is None: + return None + if unit == "ms": + return value + if unit in {"s", "seconds"}: + return value * 1000 + raise RuntimeError(f"unsupported {metric} unit {unit!r} in {path}") + + +def parse_result( + arm: BenchmarkArm, + scenario: ScenarioDefinition, + load_profile: str, + oha_path: Path | None, + aiperf_path: Path, + before_stats: dict[str, object], + after_stats: dict[str, object], +) -> BenchmarkResult: + """Combine one AIPerf run, an optional oha baseline, and routing counter deltas.""" + oha_rps = oha_p50 = oha_p99 = None + if oha_path is not None: + oha = read_json_object(oha_path) + success, _unit = _metric(oha, "summary", "successRate") + if success != 1: + raise RuntimeError(f"oha success rate for {arm.model} was {success}; see {oha_path}") + oha_rps, _unit = _metric(oha, "summary", "requestsPerSec") + p50, _unit = _metric(oha, "latencyPercentiles", "p50") + p99, _unit = _metric(oha, "latencyPercentiles", "p99") + oha_p50 = p50 * 1000 if p50 is not None else None + oha_p99 = p99 * 1000 if p99 is not None else None + aiperf = read_json_object(aiperf_path) + errors, _unit = _metric(aiperf, "error_request_count", "avg") + error_count = errors or 0.0 + measured_requests, _unit = _metric(aiperf, "request_count", "avg") + error_rate = ( + error_count / measured_requests + if measured_requests is not None and measured_requests > 0 + else 1.0 + ) + expectation_met = ( + scenario.expected_error_rate_min <= error_rate <= scenario.expected_error_rate_max + ) + throughput, _unit = _metric(aiperf, "request_throughput", "avg") + output_throughput, _unit = _metric(aiperf, "output_token_throughput", "avg") + user_throughput, _unit = _metric(aiperf, "e2e_output_token_throughput", "p50") + aggregate = aiperf.get("metrics") + throughput_stats = ( + aggregate.get("request_throughput_avg") if isinstance(aggregate, dict) else None + ) + routing = stats_delta(before_stats, after_stats) + return BenchmarkResult( + algorithm=arm.label, + model=arm.model, + scenario=scenario.id, + scenario_description=scenario.description, + scenario_group=scenario.group, + load_profile=load_profile, + expected_behavior=scenario.expected, + expected_error_rate_min=scenario.expected_error_rate_min, + expected_error_rate_max=scenario.expected_error_rate_max, + expectation_met=expectation_met, + oha_requests_per_second=oha_rps, + oha_latency_p50_ms=oha_p50, + oha_latency_p99_ms=oha_p99, + aiperf_requests_per_second=throughput, + aiperf_request_latency_p50_ms=_latency(aiperf, "request_latency", "p50", aiperf_path), + aiperf_ttft_p50_ms=_latency(aiperf, "time_to_first_token", "p50", aiperf_path), + aiperf_ttft_p99_ms=_latency(aiperf, "time_to_first_token", "p99", aiperf_path), + aiperf_itl_p50_ms=_latency(aiperf, "inter_token_latency", "p50", aiperf_path), + aiperf_output_tokens_per_second=output_throughput, + aiperf_output_tokens_per_second_per_user_p50=user_throughput, + aiperf_error_requests=error_count, + aiperf_error_rate=error_rate, + aiperf_request_throughput_cv=( + float(throughput_stats["cv"]) + if isinstance(throughput_stats, dict) + and isinstance(throughput_stats.get("cv"), int | float) + else None + ), + aiperf_request_throughput_ci_low=( + float(throughput_stats["ci_low"]) + if isinstance(throughput_stats, dict) + and isinstance(throughput_stats.get("ci_low"), int | float) + else None + ), + aiperf_request_throughput_ci_high=( + float(throughput_stats["ci_high"]) + if isinstance(throughput_stats, dict) + and isinstance(throughput_stats.get("ci_high"), int | float) + else None + ), + selected_model_calls=routing.model_calls, + selected_model_share=routing.model_share, + selected_model_errors=routing.model_errors, + classifier_calls=routing.classifier_calls, + classifier_errors=routing.classifier_errors, + classifier_latency_avg_ms=routing.classifier_latency_avg_ms, + routing_overhead_avg_ms=routing.routing_overhead_avg_ms, + bypasses_switchyard=arm.bypasses_switchyard, + ) + + +def _percent_change(value: float | None, baseline: float | None) -> float | None: + if value is None or baseline is None or baseline == 0: + return None + return (value / baseline - 1) * 100 + + +def _difference(value: float | None, baseline: float | None) -> float | None: + if value is None or baseline is None: + return None + return value - baseline + + +def _direct_baselines( + results: Sequence[BenchmarkResult], +) -> dict[tuple[str, str], BenchmarkResult]: + """Index the one direct-backend result allowed for each workload.""" + baselines = {} + for result in results: + if not result.bypasses_switchyard: + continue + key = (result.scenario, result.load_profile) + if key in baselines: + raise RuntimeError( + f"duplicate direct baseline for {result.scenario} {result.load_profile}" + ) + baselines[key] = result + return baselines + + +def compare_to_direct_backend(results: Sequence[BenchmarkResult]) -> list[BenchmarkResult]: + """Attach AIPerf deltas from the direct-backend row for each workload.""" + baselines = _direct_baselines(results) + if not baselines: + return list(results) + + compared = [] + for result in results: + if result.bypasses_switchyard: + compared.append(result) + continue + key = (result.scenario, result.load_profile) + baseline = baselines.get(key) + if baseline is None: + if result.scenario_group != "resilience": + raise RuntimeError( + f"missing direct baseline for {result.scenario} {result.load_profile}" + ) + compared.append(result) + continue + if result.aiperf_error_rate != 0 or baseline.aiperf_error_rate != 0: + compared.append(result) + continue + compared.append( + replace( + result, + aiperf_request_throughput_delta_pct=_percent_change( + result.aiperf_requests_per_second, + baseline.aiperf_requests_per_second, + ), + aiperf_request_latency_p50_delta_ms=_difference( + result.aiperf_request_latency_p50_ms, + baseline.aiperf_request_latency_p50_ms, + ), + aiperf_ttft_p50_delta_ms=_difference( + result.aiperf_ttft_p50_ms, + baseline.aiperf_ttft_p50_ms, + ), + aiperf_ttft_p50_delta_pct=_percent_change( + result.aiperf_ttft_p50_ms, + baseline.aiperf_ttft_p50_ms, + ), + aiperf_ttft_p99_delta_ms=_difference( + result.aiperf_ttft_p99_ms, + baseline.aiperf_ttft_p99_ms, + ), + aiperf_output_tokens_per_second_delta=_difference( + result.aiperf_output_tokens_per_second, + baseline.aiperf_output_tokens_per_second, + ), + aiperf_output_tokens_per_second_delta_pct=_percent_change( + result.aiperf_output_tokens_per_second, + baseline.aiperf_output_tokens_per_second, + ), + ) + ) + return compared + + +def format_metric(value: float | None) -> str: + """Render a report value without implying precision the benchmark lacks.""" + return "n/a" if value is None else f"{value:.2f}" + + +def format_delta(value: float | None, suffix: str = "") -> str: + """Render a signed direct-backend comparison.""" + return "n/a" if value is None else f"{value:+,.2f}{suffix}" + + +def write_report(config: BenchmarkConfig, results: Sequence[BenchmarkResult]) -> None: + """Write machine-readable and reviewer-readable comparison reports.""" + if not results: + raise RuntimeError("no scenario and load-profile combinations were selected") + rows = [asdict(result) for result in results] + payload = { + "generated_at": datetime.now(timezone.utc).isoformat(), + "backend": config.backend_label, + "base_url": config.base_url, + "direct_baseline": ( + asdict(config.direct_baseline) if config.direct_baseline is not None else None + ), + "concurrency": config.concurrency, + "request_count": config.request_count, + "profile_runs": config.profile_runs, + "scenario_set": config.scenario_set, + "selected_scenarios": list(config.scenarios), + "selected_load_profiles": list(config.load_profiles), + "aiperf_tokenizer": config.tokenizer, + "results": rows, + } + (config.output_dir / "report.json").write_text( + f"{json.dumps(payload, indent=2)}\n", encoding="utf-8" + ) + with (config.output_dir / "report.csv").open("w", encoding="utf-8", newline="") as output: + writer = csv.DictWriter(output, fieldnames=list(rows[0])) + writer.writeheader() + writer.writerows(rows) + + lines = ["# Routing algorithm performance", ""] + run_details = [ + f"- Backend: {config.backend_label}", + f"- AIPerf repetitions per row: {config.profile_runs}", + f"- Scenario set: {config.scenario_set}", + "- Run order: algorithms ran back-to-back within each scenario and load; all jobs " + "remained sequential", + ] + if config.direct_baseline is not None: + run_details.append( + f"- Direct baseline: `{config.direct_baseline.model}` at " + f"`{config.direct_baseline.base_url}`; this arm bypasses Switchyard" + ) + overhead_results = [ + result + for result in results + if not result.bypasses_switchyard and result.scenario_group != "resilience" + ] + if overhead_results: + write_overhead_plot( + config.output_dir / "routing-overhead.svg", + [ + OverheadPlotRow( + scenario=result.scenario, + load=result.load_profile, + route=result.algorithm, + ttft_delta_ms=result.aiperf_ttft_p50_delta_ms, + ttft_delta_pct=result.aiperf_ttft_p50_delta_pct, + token_throughput_delta=(result.aiperf_output_tokens_per_second_delta), + token_throughput_delta_pct=( + result.aiperf_output_tokens_per_second_delta_pct + ), + ) + for result in overhead_results + ], + ) + lines.extend( + ( + "## Routing overhead versus the direct backend", + "", + "The direct arm sends the same Rust-exported requests without Switchyard. These " + "deltas isolate Switchyard overhead only when both arms use the same backend " + "deployment, model, and response settings. Positive latency means the routed " + "request took longer. Negative throughput means the routed path processed less " + "work. Resilience workloads are shown later but are not compared because their " + "failure behavior differs.", + "", + ) + ) + if overhead_results: + lines.extend( + ( + "![Routing overhead plots](routing-overhead.svg)", + "", + ) + ) + if not overhead_results: + lines.append("No successful direct-backend comparisons were available.") + else: + lines.extend( + ( + "| Workload | Route | Load | Request p50 change (ms) | " + "TTFT p50 change (ms) | TTFT change (%) | Request throughput change (%) | " + "Token throughput change (tokens/s) | Token throughput change (%) |", + "|---|---|---|---:|---:|---:|---:|---:|---:|", + ) + ) + for result in overhead_results: + lines.append( + f"| {result.scenario.replace('-', ' ')} | {result.algorithm} | " + f"{result.load_profile} | " + f"{format_delta(result.aiperf_request_latency_p50_delta_ms, ' ms')} | " + f"{format_delta(result.aiperf_ttft_p50_delta_ms, ' ms')} | " + f"{format_delta(result.aiperf_ttft_p50_delta_pct, '%')} | " + f"{format_delta(result.aiperf_request_throughput_delta_pct, '%')} | " + f"{format_delta(result.aiperf_output_tokens_per_second_delta, ' tokens/s')} | " + f"{format_delta(result.aiperf_output_tokens_per_second_delta_pct, '%')} |" + ) + lines.extend( + ( + "", + "Use the confidence intervals in the detailed results before treating a small " + "throughput change as meaningful. A small negative latency or positive " + "throughput delta is run-to-run noise when those intervals overlap. Request and " + "token throughput percentage changes match when every arm emits the same fixed " + "output length; the detailed table shows the absolute token rates.", + "", + "## Workloads measured", + "", + "| Workload | Request pattern |", + "|---|---|", + ) + ) + seen_scenarios = set() + for result in overhead_results: + if result.scenario in seen_scenarios: + continue + seen_scenarios.add(result.scenario) + lines.append(f"| {result.scenario.replace('-', ' ')} | {result.scenario_description} |") + lines.extend(("", "## Run details", "", *run_details)) + lines.extend( + ( + "", + "## Throughput and latency", + "", + "| Algorithm | Scenario | Load | oha req/s | AIPerf req/s | request p50 ms | " + "TTFT p50 ms | TTFT p99 ms | ITL p50 ms | output tok/s | error rate | gate |", + "|---|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---|", + ) + ) + for result in results: + if result.scenario_group == "resilience": + continue + lines.append( + f"| {result.algorithm} | {result.scenario} | {result.load_profile} | " + f"{format_metric(result.oha_requests_per_second)} | " + f"{format_metric(result.aiperf_requests_per_second)} | " + f"{format_metric(result.aiperf_request_latency_p50_ms)} | " + f"{format_metric(result.aiperf_ttft_p50_ms)} | " + f"{format_metric(result.aiperf_ttft_p99_ms)} | " + f"{format_metric(result.aiperf_itl_p50_ms)} | " + f"{format_metric(result.aiperf_output_tokens_per_second)} | " + f"{result.aiperf_error_rate:.2%} | " + f"{'PASS' if result.expectation_met else 'FAIL'} |" + ) + if config.profile_runs > 1: + lines.extend( + ( + "", + "## Repeatability", + "", + "The script calculates variation and confidence intervals across independent " + "AIPerf runs. " + "Treat a small throughput delta as noise when these intervals overlap.", + "", + "| Algorithm | Scenario | Load | req/s average | req/s CV | req/s 95% CI |", + "|---|---|---|---:|---:|---:|", + ) + ) + for result in results: + if result.scenario_group == "resilience": + continue + confidence_interval = ( + "n/a" + if result.aiperf_request_throughput_ci_low is None + or result.aiperf_request_throughput_ci_high is None + else f"{result.aiperf_request_throughput_ci_low:.2f}–" + f"{result.aiperf_request_throughput_ci_high:.2f}" + ) + coefficient_of_variation = ( + "n/a" + if result.aiperf_request_throughput_cv is None + else f"{result.aiperf_request_throughput_cv:.2%}" + ) + lines.append( + f"| {result.algorithm} | {result.scenario} | {result.load_profile} | " + f"{format_metric(result.aiperf_requests_per_second)} | " + f"{coefficient_of_variation} | {confidence_interval} |" + ) + lines.extend( + ( + "", + "## Routing behavior", + "", + "| Algorithm | Scenario | Load | selected target calls | selected target share | target errors | " + "classifier calls | classifier errors | classifier avg ms | " + "Switchyard routing time avg ms |", + "|---|---|---|---|---|---|---:|---:|---:|---:|", + ) + ) + for result in results: + if result.bypasses_switchyard: + continue + lines.append( + f"| {result.algorithm} | {result.scenario} | {result.load_profile} | " + f"`{result.selected_model_calls}` | `{result.selected_model_share}` | " + f"`{result.selected_model_errors}` | " + f"{result.classifier_calls} | {result.classifier_errors} | " + f"{format_metric(result.classifier_latency_avg_ms)} | " + f"{format_metric(result.routing_overhead_avg_ms)} |" + ) + + resilience = [result for result in results if result.scenario_group == "resilience"] + if resilience: + lines.extend( + ( + "", + "## Resilience", + "", + "| Algorithm | Scenario | Load | client error rate | expected range | gate | " + "target errors | Expected behavior |", + "|---|---|---|---:|---|---|---|---|", + ) + ) + for result in resilience: + if result.bypasses_switchyard: + continue + lines.append( + f"| {result.algorithm} | {result.scenario} | {result.load_profile} | " + f"{result.aiperf_error_rate:.2%} | " + f"{result.expected_error_rate_min:.2%}–{result.expected_error_rate_max:.2%} | " + f"{'PASS' if result.expectation_met else 'FAIL'} | " + f"`{result.selected_model_errors}` | " + f"{result.expected_behavior} |" + ) + lines.extend( + ( + "", + "oha runs only for the fixed short-interactive baseline. AIPerf replays the same " + "Rust-exported request payload for streaming, session-aware LLM metrics. Compare " + "algorithms within the same scenario and load row.", + "", + "The local scenario backend isolates Switchyard routing and protocol overhead. It " + "does not predict model capacity. Repeat the same command against routes backed by " + "one real deployment to include tokenization, model execution, and provider queuing.", + "", + ) + ) + (config.output_dir / "report.md").write_text("\n".join(lines), encoding="utf-8") + + +def run_oha( + config: BenchmarkConfig, + label: str, + scenario: ScenarioDefinition, + output_dir: Path, +) -> Path: + """Run oha only for the fixed short-interactive baseline.""" + inputs = read_json_object(scenario.input_file) + data = inputs.get("data") + try: + payload = data[0]["payloads"][0] # type: ignore[index] + except (IndexError, KeyError, TypeError) as error: + raise RuntimeError(f"invalid AIPerf input file {scenario.input_file}") from error + if not isinstance(payload, dict): + raise RuntimeError(f"invalid AIPerf payload in {scenario.input_file}") + body = dict(payload) + body["stream"] = False + request_path = output_dir / f"{label}-request.json" + result_path = output_dir / f"{label}.json" + request_path.write_text(json.dumps(body), encoding="utf-8") + run_checked( + f"oha {label}", + [ + config.oha_bin, + "-n", + str(config.request_count), + "-c", + str(config.concurrency), + "--latency-correction", + "--no-tui", + "--method", + "POST", + "-T", + "application/json", + "-D", + str(request_path), + "--output-format", + "json", + "--output", + str(result_path), + f"{config.base_url.rstrip('/')}/v1/chat/completions", + ], + output_dir / f"{label}.log", + ) + return result_path + + +def materialize_aiperf_input( + scenario: ScenarioDefinition, + output_path: Path, + minimum_request_count: int, + namespace: str = "load", +) -> Path: + """Repeat Rust-owned sessions with unique ids so AIPerf never wraps the dataset.""" + if minimum_request_count > MAX_MATERIALIZED_REQUESTS: + raise RuntimeError( + f"AIPerf input needs {minimum_request_count} requests; maximum is " + f"{MAX_MATERIALIZED_REQUESTS}" + ) + document = read_json_object(scenario.input_file) + data = document.get("data") + if not isinstance(data, list) or not data: + raise RuntimeError(f"AIPerf input has no sessions: {scenario.input_file}") + + validated = [] + requests_per_replica = 0 + bytes_per_replica = 0 + for raw_session in data: + if not isinstance(raw_session, dict): + raise RuntimeError(f"invalid AIPerf session in {scenario.input_file}") + session_id = raw_session.get("session_id") + payloads = raw_session.get("payloads") + if not isinstance(session_id, str) or not isinstance(payloads, list) or not payloads: + raise RuntimeError(f"invalid AIPerf session in {scenario.input_file}") + validated.append((raw_session, session_id, len(payloads))) + requests_per_replica += len(payloads) + bytes_per_replica += len(json.dumps(raw_session).encode()) + 64 + replica_count = math.ceil(minimum_request_count / requests_per_replica) + estimated_bytes = replica_count * bytes_per_replica + if estimated_bytes > MAX_MATERIALIZED_BYTES: + raise RuntimeError( + f"AIPerf input would use about {estimated_bytes} bytes; maximum is " + f"{MAX_MATERIALIZED_BYTES}" + ) + + sessions = [] + request_count = 0 + replica = 0 + for _replica_index in range(replica_count): + for raw_session, session_id, payload_count in validated: + sessions.append( + { + **raw_session, + "session_id": f"{session_id}-{namespace}-{replica:06d}", + } + ) + request_count += payload_count + replica += 1 + if request_count >= minimum_request_count: + break + if request_count >= minimum_request_count: + break + output_path.write_text(json.dumps({"data": sessions}), encoding="utf-8") + return output_path + + +def profile_request_count(config: BenchmarkConfig, profile: dict[str, object]) -> int: + """Return a conservative request bound for one AIPerf schedule.""" + if profile.get("kind") != "traffic_burst": + return config.request_count + duration = profile.get("duration_seconds") + points = profile.get("points") + if not isinstance(duration, int) or not isinstance(points, list): + raise RuntimeError("traffic-burst profile is missing its duration or points") + parsed_points = [] + for point in points: + if not isinstance(point, dict): + raise RuntimeError("traffic-burst profile has an invalid point") + time_s = point.get("time_s") + multiplier = point.get("rate_multiplier") + if not isinstance(time_s, int | float) or not isinstance(multiplier, int | float): + raise RuntimeError("traffic-burst profile has an invalid point") + parsed_points.append((float(time_s), float(multiplier))) + request_multiplier_seconds = sum( + (right_time - left_time) * (left_rate + right_rate) / 2 + for (left_time, left_rate), (right_time, right_rate) in zip( + parsed_points, parsed_points[1:], strict=False + ) + ) + return math.ceil(request_multiplier_seconds * (config.request_rate or config.concurrency)) + + +def aiperf_timeout_seconds( + config: BenchmarkConfig, + scenario: ScenarioDefinition, + profile: dict[str, object], + concurrency: int | None, +) -> int: + """Bound one process by its schedule and per-request timeout.""" + request_timeout = 1 if scenario.id == "client-cancellation" else 30 + if profile.get("kind") == "traffic_burst": + duration = profile.get("duration_seconds") + if not isinstance(duration, int): + raise RuntimeError(f"traffic-burst profile has no duration for {scenario.id}") + workload_seconds = duration + request_timeout + else: + effective_concurrency = concurrency or config.concurrency + workload_seconds = math.ceil(config.request_count / effective_concurrency) * request_timeout + return AIPERF_LIFECYCLE_ALLOWANCE_SECONDS + workload_seconds + + +def run_aiperf( + config: BenchmarkConfig, + arm: BenchmarkArm, + scenario: ScenarioDefinition, + profile: dict[str, object], + output_dir: Path, + artifact_label: str, + concurrency: int | None = None, +) -> Path: + """Run independent AIPerf repetitions using Rust-exported inputs-json datasets.""" + artifact_root = output_dir / artifact_label + effective_concurrency = concurrency or config.concurrency + load_arguments: list[str] = [] + kind = profile.get("kind") + if kind == "traffic_burst": + points = profile.get("points") + if not isinstance(points, list): + raise RuntimeError(f"traffic-burst profile has no points for {scenario.id}") + base_rate = config.request_rate or float(config.concurrency) + rate_points = [] + for point in points: + if not isinstance(point, dict): + raise RuntimeError(f"invalid traffic-burst point for {scenario.id}") + rate_points.append( + { + "time_s": point["time_s"], + "qps": base_rate * float(point["rate_multiplier"]), + } + ) + series_path = output_dir / f"{artifact_label}-request-rate.json" + series_path.write_text( + f"{json.dumps({'points': rate_points}, indent=2)}\n", encoding="utf-8" + ) + load_arguments.extend( + ("--request-rate-series", str(series_path), "--arrival-pattern", "constant") + ) + load_arguments.extend(("--benchmark-duration", str(profile.get("duration_seconds")))) + load_arguments.extend(("--concurrency", str(config.concurrency))) + else: + load_arguments.extend(("--concurrency", str(effective_concurrency))) + load_arguments.extend(("--request-count", str(config.request_count))) + + exports: list[Path] = [] + for trial in range(1, config.profile_runs + 1): + run_label = f"{artifact_label}-run-{trial:02d}" + input_path = materialize_aiperf_input( + scenario, + output_dir / f"{run_label}-inputs.json", + profile_request_count(config, profile), + namespace=f"{artifact_label}-run-{trial:02d}", + ) + command = [ + config.aiperf_bin, + "profile", + "--model", + arm.model, + "--url", + arm.base_url, + "--endpoint-type", + "chat", + "--streaming", + "--tokenizer", + config.tokenizer, + "--custom-dataset-type", + "inputs-json", + "--input-file", + str(input_path), + "--session-header", + "x-switchyard-session-id", + "--random-seed", + "42", + "--num-profile-runs", + "1", + "--record-processors", + "1", + "--request-timeout-seconds", + "1" if scenario.id == "client-cancellation" else "30", + "--ui", + "none", + *load_arguments, + ] + if arm.api_key_env is not None: + command.extend(("--api-key", f"${{{arm.api_key_env}}}")) + trial_root = ( + artifact_root / "profile_runs" / f"run_{trial:04d}" + if config.profile_runs > 1 + else artifact_root + ) + exports.append( + run_profile( + command, + output_dir / f"{run_label}.log", + trial_root, + aiperf_timeout_seconds(config, scenario, profile, concurrency), + ) + ) + if len(exports) == 1: + return exports[0] + aggregate_path: Path = aggregate_exports( + exports, + artifact_root / "aggregate" / "profile_export_aiperf_aggregate.json", + ) + return aggregate_path + + +def run_benchmark(config: BenchmarkConfig) -> Path: + """Run selected scenarios for every algorithm and return the report directory.""" + if not config.models: + raise RuntimeError("at least one algorithm model is required") + if config.profile_runs > 10: + raise RuntimeError("--profile-runs must be between 1 and 10") + validate_aiperf_version(config.aiperf_bin) + arms = [BenchmarkArm(label, model, config.base_url, False) for label, model in config.models] + if config.direct_baseline is not None: + arms.insert( + 0, + BenchmarkArm( + "direct-backend", + config.direct_baseline.model, + config.direct_baseline.base_url, + True, + config.direct_baseline.api_key_env, + ), + ) + labels = [arm.label for arm in arms] + if len(labels) != len(set(labels)): + raise RuntimeError("algorithm labels must be unique") + config.output_dir.mkdir(parents=True, exist_ok=False) + oha_dir = config.output_dir / "oha" + aiperf_dir = config.output_dir / "aiperf" + oha_dir.mkdir() + aiperf_dir.mkdir() + results = [] + definition_sets = [export_scenarios(config, index, arm.model) for index, arm in enumerate(arms)] + expected_ids = [scenario.id for scenario in definition_sets[0]] + for definitions in definition_sets[1:]: + if [scenario.id for scenario in definitions] != expected_ids: + raise RuntimeError("scenario exports differ between algorithms") + + for scenario_index, baseline_scenario in enumerate(definition_sets[0]): + for profile in selected_profiles(config, baseline_scenario): + profile_id = str(profile["id"]) + concurrencies: list[int | None] = [None] + if profile.get("kind") == "concurrency_knee": + steps = profile.get("concurrency_steps") + if not isinstance(steps, list) or not all(isinstance(step, int) for step in steps): + raise RuntimeError(f"invalid concurrency knee for {baseline_scenario.id}") + concurrencies = sorted( + set( + [step for step in steps if step <= config.concurrency] + + [config.concurrency] + ) + ) + for concurrency in concurrencies: + load_label = profile_id if concurrency is None else f"{profile_id}@{concurrency}" + for index, arm in enumerate(arms): + scenario = definition_sets[index][scenario_index] + if arm.bypasses_switchyard and scenario.group == "resilience": + continue + artifact_label = ( + f"algorithm-{index:02d}-{scenario.id}-{load_label.replace('@', '-')}" + ) + oha_path = None + if ( + not arm.bypasses_switchyard + and scenario.id == "short-interactive" + and profile_id == "fixed" + ): + oha_path = run_oha( + config, + artifact_label, + scenario, + oha_dir, + ) + reset_scenario_backend(config.scenario_backend_reset_url) + before = ( + {} + if arm.bypasses_switchyard + else capture_stats( + config.base_url, + aiperf_dir / f"{artifact_label}-stats-before.json", + ) + ) + aiperf_path = run_aiperf( + config, + arm, + scenario, + profile, + aiperf_dir, + artifact_label, + concurrency, + ) + after = ( + {} + if arm.bypasses_switchyard + else capture_stats( + config.base_url, + aiperf_dir / f"{artifact_label}-stats-after.json", + ) + ) + results.append( + parse_result( + arm, + scenario, + load_label, + oha_path, + aiperf_path, + before, + after, + ) + ) + results = compare_to_direct_backend(results) + write_report(config, results) + failures = [result for result in results if not result.expectation_met] + if failures: + raise RuntimeError( + f"{len(failures)} benchmark row(s) missed their error-rate gate; " + f"see {config.output_dir / 'report.md'}" + ) + return config.output_dir + + +def default_output_dir() -> Path: + """Choose a new timestamped directory under the repository.""" + repo_root = Path(__file__).resolve().parents[1] + stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + return repo_root / "routing-benchmark-results" / stamp + + +def main(argv: Sequence[str] | None = None) -> int: + """Parse arguments, run the comparison, and print one actionable failure.""" + command = parser() + args = command.parse_args(argv) + if (args.direct_base_url is None) != (args.direct_model is None): + command.error("--direct-base-url and --direct-model must be used together") + if args.direct_api_key_env is not None and args.direct_base_url is None: + command.error("--direct-api-key-env requires --direct-base-url and --direct-model") + try: + binaries = resolve_binaries( + ( + RequiredBinary( + "oha", "OHA_BIN", args.oha_bin, "Install oha with: cargo install oha" + ), + RequiredBinary( + "AIPerf", + "AIPERF_BIN", + args.aiperf_bin, + "Install AIPerf with: uv tool install --python 3.12 'aiperf==0.11.0'", + ), + RequiredBinary( + "switchyard-soak", + "SWITCHYARD_SOAK_BIN", + args.soak_bin, + "Build the scenario exporter with: cargo build --release -p switchyard-soak", + ), + ) + ) + output_dir = run_benchmark( + BenchmarkConfig( + base_url=args.base_url, + models=tuple(args.models), + concurrency=args.concurrency, + request_count=args.request_count, + tokenizer=args.tokenizer, + scenario_set=args.scenario_set, + scenarios=tuple(args.scenario), + load_profiles=tuple(args.load_profile or ["fixed"]), + profile_runs=args.profile_runs, + prompt_bytes=args.prompt_bytes, + max_output_tokens=args.max_output_tokens, + context_window_tokens=args.context_window_tokens, + request_rate=args.request_rate, + backend_label=args.backend_label, + output_dir=(args.output_dir or default_output_dir()).resolve(), + oha_bin=binaries["oha"], + aiperf_bin=binaries["AIPerf"], + soak_bin=binaries["switchyard-soak"], + scenario_backend_reset_url=args.scenario_backend_reset_url, + direct_baseline=( + DirectBaseline( + args.direct_base_url, + args.direct_model, + args.direct_api_key_env, + ) + if args.direct_base_url is not None and args.direct_model is not None + else None + ), + ) + ) + except (OSError, RuntimeError) as error: + print(f"routing algorithm benchmark failed: {error}", file=sys.stderr) + return 1 + print(f"Routing algorithm benchmark passed; report: {output_dir / 'report.md'}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/local_soak_test.toml b/scripts/local_soak_test.toml new file mode 100644 index 000000000..967095094 --- /dev/null +++ b/scripts/local_soak_test.toml @@ -0,0 +1,57 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Local routes used by run_local_soak_test.py; no provider credentials are required. + +schema_version = 1 + +[llm_clients.mock] +format = "openai_chat" +base_url = "http://127.0.0.1:8100/v1" +max_retries = 10 + +[targets.classifier] +id = "mock/classifier" +llm_client = "mock" + +[targets.strong] +id = "mock/strong" +llm_client = "mock" + +[targets.weak] +id = "mock/weak" +llm_client = "mock" + +[routes.noop] +id = "switchyard/noop" +type = "noop" + +[routes.random] +id = "switchyard/random" +type = "random" +targets = ["weak", "strong"] +weights = [1, 0] +seed = 42 + +[routes.passthrough] +id = "switchyard/passthrough" +type = "passthrough" +target = "weak" + +[routes.classifier] +id = "switchyard/classifier" +type = "llm_classifier" +mode = "capability" +classifier_target = "classifier" +strong_target = "strong" +weak_target = "weak" +base_threshold = 1.0 +max_output_tokens = 1 + +[routes.stage] +id = "switchyard/stage" +type = "stage_router" +capable_target = "strong" +efficient_target = "weak" +picker = "efficient_first" +confidence_threshold = 1.0 diff --git a/scripts/routing_overhead_plot.py b/scripts/routing_overhead_plot.py new file mode 100644 index 000000000..65c6fd2a8 --- /dev/null +++ b/scripts/routing_overhead_plot.py @@ -0,0 +1,291 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Render routing-overhead results as a dependency-free SVG heatmap.""" + +import math +from collections.abc import Sequence +from dataclasses import dataclass +from html import escape +from pathlib import Path +from typing import TypeGuard + + +@dataclass(frozen=True) +class OverheadPlotRow: + """One route and workload comparison against the direct backend.""" + + scenario: str + load: str + route: str + ttft_delta_ms: float | None + ttft_delta_pct: float | None + token_throughput_delta: float | None + token_throughput_delta_pct: float | None + + +@dataclass(frozen=True) +class _Panel: + metric: str + title: str + subtitle: str + primary_suffix: str + primary_values: tuple[float | None, ...] + percent_values: tuple[float | None, ...] + higher_is_better: bool + + +_BACKGROUND = (255, 255, 255) +_BETTER = (42, 111, 174) +_WORSE = (197, 52, 52) +_NEUTRAL = "#f3f5f7" + + +def _finite(value: float | None) -> TypeGuard[float]: + return value is not None and math.isfinite(value) + + +def _fill(value: float | None, maximum: float, higher_is_better: bool) -> str: + if not _finite(value) or value == 0: + return _NEUTRAL + worse = value < 0 if higher_is_better else value > 0 + target = _WORSE if worse else _BETTER + intensity = 0.16 + 0.64 * min(abs(value) / maximum, 1.0) + channels = ( + round(background + (foreground - background) * intensity) + for background, foreground in zip(_BACKGROUND, target, strict=True) + ) + red, green, blue = channels + return f"#{red:02x}{green:02x}{blue:02x}" + + +def _format_value(value: float | None, suffix: str) -> str: + if not _finite(value): + return "n/a" + return f"{value:+,.2f}{suffix}" + + +def _text( + x: float, + y: float, + value: str, + anchor: str = "start", + size: int = 13, + weight: int = 400, + fill: str = "#1d2733", +) -> str: + return ( + f'{escape(value)}' + ) + + +def write_overhead_plot(path: Path, rows: Sequence[OverheadPlotRow]) -> None: + """Write an annotated SVG overview for every supplied route and workload.""" + if not rows: + raise ValueError("an overhead plot requires at least one row") + + workloads = list(dict.fromkeys((row.scenario, row.load) for row in rows)) + routes = list(dict.fromkeys(row.route for row in rows)) + by_key: dict[tuple[str, str, str], OverheadPlotRow] = {} + for row in rows: + key = (row.scenario, row.load, row.route) + if key in by_key: + raise RuntimeError( + f"duplicate overhead plot row for {row.scenario}, {row.load}, and {row.route}" + ) + by_key[key] = row + + ordered_rows = [ + by_key.get((scenario, load, route)) for scenario, load in workloads for route in routes + ] + panels = ( + _Panel( + "ttft_p50_delta_ms", + "TTFT p50 overhead", + "Each cell shows the absolute change in ms and the percent change; lower is better", + " ms", + tuple(row.ttft_delta_ms if row is not None else None for row in ordered_rows), + tuple(row.ttft_delta_pct if row is not None else None for row in ordered_rows), + False, + ), + _Panel( + "output_tokens_per_second_delta", + "Output token throughput overhead", + "Each cell shows the absolute change in tokens/s and the percent change; higher is better", + " tokens/s", + tuple(row.token_throughput_delta if row is not None else None for row in ordered_rows), + tuple( + row.token_throughput_delta_pct if row is not None else None for row in ordered_rows + ), + True, + ), + ) + + label_width = 250 + cell_width = 170 + row_height = 50 + panel_header_height = 76 + panel_gap = 52 + top = 112 + right = 24 + panel_height = panel_header_height + len(workloads) * row_height + width = label_width + len(routes) * cell_width + right + height = top + len(panels) * panel_height + panel_gap + 28 + center = width / 2 + legend_x = center - 260 + + svg = [ + ( + f'' + ), + 'Routing overhead versus the direct backend', + ( + 'Two annotated heatmaps compare median ' + "time to first token in milliseconds and output token throughput in tokens per " + "second for each route and workload. Every cell also gives the percent change. Red " + "cells are worse than the direct backend and blue cells are better. Darker cells " + "show a larger percent change within each heatmap." + ), + f'', + _text( + center, + 31, + "Routing overhead versus the direct backend", + anchor="middle", + size=22, + weight=700, + ), + _text( + center, + 55, + "Measures Switchyard overhead when both arms use the same deployment, model, and settings", + anchor="middle", + size=13, + fill="#596777", + ), + f'', + _text(legend_x + 24, 84, "worse", size=12, fill="#596777"), + f'', + _text(legend_x + 104, 84, "better", size=12, fill="#596777"), + f'', + _text( + legend_x + 187, + 84, + "no change / unavailable; darker = larger % change", + size=12, + fill="#596777", + ), + ] + + for panel_index, panel in enumerate(panels): + panel_y = top + panel_index * (panel_height + panel_gap) + svg.append(_text(center, panel_y + 18, panel.title, anchor="middle", size=16, weight=650)) + svg.append( + _text( + center, + panel_y + 39, + panel.subtitle, + anchor="middle", + size=12, + fill="#596777", + ) + ) + svg.append( + _text( + label_width / 2, + panel_y + 66, + "Workload / load", + anchor="middle", + size=12, + weight=600, + ) + ) + for route_index, route in enumerate(routes): + cell_x = label_width + route_index * cell_width + svg.append( + _text( + cell_x + cell_width / 2, + panel_y + 66, + route.replace("-", " ").replace("_", " "), + anchor="middle", + size=12, + weight=600, + ) + ) + + finite_values = [abs(value) for value in panel.percent_values if _finite(value)] + maximum = max(finite_values, default=1.0) or 1.0 + row_y = panel_y + panel_header_height + for workload_index, (scenario, load) in enumerate(workloads): + y = row_y + workload_index * row_height + svg.append( + _text( + label_width / 2, + y + 21, + scenario.replace("-", " "), + anchor="middle", + size=12, + fill="#344150", + ) + ) + svg.append( + _text( + label_width / 2, + y + 38, + load.replace("-", " "), + anchor="middle", + size=11, + fill="#697786", + ) + ) + for route_index in range(len(routes)): + value_index = workload_index * len(routes) + route_index + primary_value = panel.primary_values[value_index] + percent_value = panel.percent_values[value_index] + x = label_width + route_index * cell_width + cell_title = ( + f"{scenario}, {load}, {routes[route_index]}: " + f"{_format_value(primary_value, panel.primary_suffix)} " + f"({_format_value(percent_value, '%')})" + ) + svg.append( + f'' + ) + svg.append(f"{escape(cell_title)}") + svg.append( + f'' + ) + svg.append( + _text( + x + cell_width / 2, + y + 21, + _format_value(primary_value, panel.primary_suffix), + anchor="middle", + size=12, + weight=600, + ) + ) + svg.append( + _text( + x + cell_width / 2, + y + 38, + f"({_format_value(percent_value, '%')})", + anchor="middle", + size=11, + fill="#44515f", + ) + ) + svg.append("") + + svg.append("") + path.write_text("\n".join(svg) + "\n", encoding="utf-8") diff --git a/scripts/run_local_soak_test.py b/scripts/run_local_soak_test.py new file mode 100755 index 000000000..32decdae3 --- /dev/null +++ b/scripts/run_local_soak_test.py @@ -0,0 +1,369 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Run every Switchyard route with the local scenario backend and load tools.""" + +import argparse +import json +import os +import shutil +import subprocess +import sys +import time +import urllib.error +import urllib.request +from collections.abc import Sequence +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import TextIO + +from benchmark_routing_algorithms import ( + BenchmarkConfig, + DirectBaseline, + RequiredBinary, + positive_int, + resolve_binaries, + run_benchmark, + run_checked, +) + +ROUTES = ( + ("noop", "switchyard/noop"), + ("random", "switchyard/random"), + ("passthrough", "switchyard/passthrough"), + ("llm_classifier", "switchyard/classifier"), + ("stage_router", "switchyard/stage"), +) +MOCK_PORT = 8100 + + +@dataclass +class Child: + """A child process and the log file that must stay open while it runs.""" + + name: str + process: subprocess.Popen[str] + log: TextIO + log_path: Path + + +def nonnegative_int(value: str) -> int: + """Parse a command-line integer that may be zero.""" + parsed = int(value) + if parsed < 0: + raise argparse.ArgumentTypeError("must be zero or greater") + return parsed + + +def tcp_port(value: str) -> int: + """Parse a valid nonzero TCP port.""" + parsed = positive_int(value) + if parsed > 65_535: + raise argparse.ArgumentTypeError("must be between 1 and 65535") + return parsed + + +def parser() -> argparse.ArgumentParser: + """Build the plain-English command-line interface.""" + command = argparse.ArgumentParser( + description=( + "Start the request-aware scenario backend and a local Switchyard server, send one " + "request through every route, then run oha, AIPerf, and switchyard-soak." + ), + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + command.add_argument( + "--duration", + default="10s", + help="time to run switchyard-soak; use an s, m, or h suffix", + ) + command.add_argument( + "--concurrency", + type=positive_int, + default=4, + help="concurrent requests used by oha, AIPerf, and the soak run", + ) + command.add_argument( + "--request-count", + type=positive_int, + default=100, + help="measured requests each load tool sends for each routing algorithm", + ) + command.add_argument( + "--mock-latency-ms", + type=nonnegative_int, + default=40, + help="latency before the local scenario backend sends its first token", + ) + command.add_argument( + "--mock-token-latency-ms", + type=nonnegative_int, + default=1, + help="latency the local scenario backend adds between streamed output tokens", + ) + command.add_argument( + "--server-port", + type=tcp_port, + default=4000, + help="local TCP port used by switchyard-server", + ) + command.add_argument( + "--output-dir", + type=Path, + help="new directory for generated config, logs, and tool results", + ) + return command + + +def start_child(name: str, command: Sequence[str], log_path: Path) -> Child: + """Start one long-running process and retain its combined log.""" + print(f"Starting {name}; log: {log_path}") + log = log_path.open("w", encoding="utf-8") + try: + process = subprocess.Popen( + list(command), + stdout=log, + stderr=subprocess.STDOUT, + text=True, + ) + except OSError: + log.close() + raise + return Child(name=name, process=process, log=log, log_path=log_path) + + +def stop_child(child: Child) -> None: + """Stop one child and close its log, escalating to kill after five seconds.""" + if child.process.poll() is None: + child.process.terminate() + try: + child.process.wait(timeout=5) + except subprocess.TimeoutExpired: + child.process.kill() + child.process.wait(timeout=5) + child.log.close() + + +def wait_for_health(child: Child, url: str, expected_status: str | None = None) -> None: + """Wait up to 30 seconds for a child process's health endpoint.""" + for _ in range(120): + if child.process.poll() is not None: + raise RuntimeError( + f"{child.name} exited with status {child.process.returncode}; see {child.log_path}" + ) + try: + with urllib.request.urlopen(url, timeout=1) as response: + body = response.read() + document = json.loads(body) if expected_status is not None else None + status = document.get("status") if isinstance(document, dict) else None + if response.status == 200 and ( + expected_status is None or status == expected_status + ): + return + except (OSError, urllib.error.URLError, json.JSONDecodeError): + pass + time.sleep(0.25) + raise RuntimeError(f"{child.name} did not become healthy at {url}; see {child.log_path}") + + +def check_routes(base_url: str, output_path: Path) -> None: + """Send one HTTP request through every configured route.""" + records = [] + for _algorithm, route in ROUTES: + body = json.dumps( + { + "model": route, + "messages": [{"role": "user", "content": "Reply with exactly OK."}], + "max_tokens": 8, + "stream": False, + } + ).encode() + request = urllib.request.Request( + f"{base_url}/v1/chat/completions", + data=body, + headers={"content-type": "application/json"}, + method="POST", + ) + with urllib.request.urlopen(request, timeout=30) as response: + records.append( + { + "route": route, + "status": response.status, + "selected_model": response.headers.get("x-model-router-selected-model"), + "response": json.loads(response.read()), + } + ) + output_path.write_text( + "".join(f"{json.dumps(record)}\n" for record in records), encoding="utf-8" + ) + + +def default_output_dir(repo_root: Path) -> Path: + """Choose a new timestamped directory under the repository.""" + stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + return repo_root / "local-soak-test-results" / stamp + + +def run_local_soak_test(args: argparse.Namespace) -> Path: + """Run the complete local soak test and return its output directory.""" + repo_root = Path(__file__).resolve().parents[1] + rust_build = ( + "Build the Rust commands with: cargo build --release -p switchyard-server " + "-p switchyard-soak --bins --example switchyard-soak-mock" + ) + binaries = resolve_binaries( + ( + RequiredBinary( + "switchyard-server", + "SWITCHYARD_SERVER_BIN", + os.environ.get( + "SWITCHYARD_SERVER_BIN", + str(repo_root / "target/release/switchyard-server"), + ), + rust_build, + ), + RequiredBinary( + "switchyard-soak", + "SWITCHYARD_SOAK_BIN", + os.environ.get( + "SWITCHYARD_SOAK_BIN", str(repo_root / "target/release/switchyard-soak") + ), + rust_build, + ), + RequiredBinary( + "scenario backend", + "SWITCHYARD_SOAK_MOCK_BIN", + os.environ.get( + "SWITCHYARD_SOAK_MOCK_BIN", + str(repo_root / "target/release/examples/switchyard-soak-mock"), + ), + rust_build, + ), + RequiredBinary( + "oha", + "OHA_BIN", + os.environ.get("OHA_BIN", "oha"), + "Install oha with: cargo install oha", + ), + RequiredBinary( + "AIPerf", + "AIPERF_BIN", + os.environ.get("AIPERF_BIN", "aiperf"), + "Install AIPerf with: uv tool install --python 3.12 'aiperf==0.11.0'", + ), + ) + ) + server_bin = binaries["switchyard-server"] + soak_bin = binaries["switchyard-soak"] + mock_bin = binaries["scenario backend"] + oha_bin = binaries["oha"] + aiperf_bin = binaries["AIPerf"] + + output_dir = (args.output_dir or default_output_dir(repo_root)).resolve() + output_dir.mkdir(parents=True, exist_ok=False) + config_path = output_dir / "routes.toml" + shutil.copy2(repo_root / "scripts/local_soak_test.toml", config_path) + + run_checked( + "switchyard-server config validation", + [server_bin, "--config", str(config_path), "--dry-run"], + output_dir / "config-validation.log", + ) + + children: list[Child] = [] + try: + mock = start_child( + "scenario backend", + [ + mock_bin, + "--port", + str(MOCK_PORT), + "--latency-ms", + str(args.mock_latency_ms), + "--token-latency-ms", + str(args.mock_token_latency_ms), + ], + output_dir / "scenario-backend.log", + ) + children.append(mock) + wait_for_health(mock, f"http://127.0.0.1:{MOCK_PORT}/health") + + server = start_child( + "switchyard-server", + [server_bin, "--config", str(config_path), "--port", str(args.server_port)], + output_dir / "switchyard-server.log", + ) + children.append(server) + base_url = f"http://127.0.0.1:{args.server_port}" + wait_for_health(server, f"{base_url}/health", "ok") + check_routes(base_url, output_dir / "route-checks.jsonl") + + run_benchmark( + BenchmarkConfig( + base_url=base_url, + models=ROUTES, + concurrency=args.concurrency, + request_count=args.request_count, + backend_label=( + "request-aware local backend with " + f"{args.mock_latency_ms} ms TTFT and " + f"{args.mock_token_latency_ms} ms per output token" + ), + output_dir=output_dir / "routing-benchmark", + oha_bin=oha_bin, + aiperf_bin=aiperf_bin, + soak_bin=soak_bin, + profile_runs=1, + scenario_backend_reset_url=f"http://127.0.0.1:{MOCK_PORT}/reset", + direct_baseline=DirectBaseline( + base_url=f"http://127.0.0.1:{MOCK_PORT}", + model="mock/weak", + ), + ) + ) + + run_checked( + "switchyard-soak API and streaming requests", + [ + soak_bin, + "--base-url", + base_url, + "--model", + "switchyard/passthrough", + "--duration", + args.duration, + "--concurrency", + str(args.concurrency), + "--report-interval", + "2", + "--invalid-canary-interval", + "0", + "--server-pid", + str(server.process.pid), + "--results-dir", + str(output_dir / "soak"), + ], + output_dir / "soak.log", + ) + finally: + for child in reversed(children): + stop_child(child) + + return output_dir + + +def main(argv: Sequence[str] | None = None) -> int: + """Parse arguments, run the local soak test, and print one actionable failure.""" + args = parser().parse_args(argv) + try: + output_dir = run_local_soak_test(args) + except (OSError, RuntimeError) as error: + print(f"local soak test failed: {error}", file=sys.stderr) + return 1 + print(f"Local soak test passed; results: {output_dir}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_aiperf_runner.py b/tests/test_aiperf_runner.py new file mode 100644 index 000000000..71f5fafa7 --- /dev/null +++ b/tests/test_aiperf_runner.py @@ -0,0 +1,157 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import json +import sys +import time + +import pytest + +import scripts.aiperf_runner as aiperf_runner +from scripts.aiperf_runner import aggregate_exports, run_profile, validate_aiperf_version + + +def _write_stubborn_worker(worker) -> None: + worker.write_text( + """import signal +import sys +import time +from pathlib import Path + +signal.signal(signal.SIGTERM, signal.SIG_IGN) +heartbeat = Path(sys.argv[1]) +while True: + heartbeat.write_text(str(time.monotonic())) + time.sleep(0.05) +""" + ) + + +def _assert_heartbeat_stopped(heartbeat) -> None: + heartbeat_before = heartbeat.read_text() + time.sleep(0.2) + assert heartbeat.read_text() == heartbeat_before + + +def test_run_profile_timeout_kills_the_worker_process_group(tmp_path, monkeypatch) -> None: + fake = tmp_path / "fake_aiperf.py" + worker = tmp_path / "worker.py" + child_pid = tmp_path / "child-pid" + heartbeat = tmp_path / "heartbeat" + _write_stubborn_worker(worker) + monkeypatch.setattr(aiperf_runner, "PROCESS_GROUP_GRACE_SECONDS", 0.2) + fake.write_text( + """import subprocess +import sys +import time +from pathlib import Path + +child = subprocess.Popen([sys.executable, sys.argv[2], sys.argv[3]]) +Path(sys.argv[1]).write_text(str(child.pid)) +time.sleep(60) +""" + ) + + started_at = time.monotonic() + with pytest.raises(RuntimeError, match="exceeded its 1-second run limit"): + run_profile( + [sys.executable, str(fake), str(child_pid), str(worker), str(heartbeat)], + tmp_path / "aiperf.log", + tmp_path / "artifacts", + timeout_seconds=1, + ) + + assert time.monotonic() - started_at < 10 + assert child_pid.read_text().isdigit() + _assert_heartbeat_stopped(heartbeat) + assert "exceeded its 1-second run limit" in (tmp_path / "aiperf.log").read_text() + + +def test_run_profile_stops_workers_after_the_leader_exits(tmp_path, monkeypatch) -> None: + fake = tmp_path / "fake_aiperf.py" + worker = tmp_path / "worker.py" + child_pid = tmp_path / "child-pid" + heartbeat = tmp_path / "heartbeat" + _write_stubborn_worker(worker) + monkeypatch.setattr(aiperf_runner, "PROCESS_GROUP_GRACE_SECONDS", 0.2) + fake.write_text( + """import subprocess +import sys +import time +from pathlib import Path + +child = subprocess.Popen([sys.executable, sys.argv[2], sys.argv[3]]) +Path(sys.argv[1]).write_text(str(child.pid)) +while not Path(sys.argv[3]).exists(): + time.sleep(0.01) +""" + ) + + with pytest.raises(RuntimeError, match="did not write profile_export_aiperf.json"): + run_profile( + [sys.executable, str(fake), str(child_pid), str(worker), str(heartbeat)], + tmp_path / "aiperf.log", + tmp_path / "artifacts", + timeout_seconds=10, + ) + + assert child_pid.read_text().isdigit() + _assert_heartbeat_stopped(heartbeat) + + +def test_process_group_probe_ignores_an_unowned_reused_group(monkeypatch) -> None: + def deny_signal(_process_group, _signal) -> None: + raise PermissionError + + monkeypatch.setattr(aiperf_runner.os, "killpg", deny_signal) + + assert not aiperf_runner._process_group_exists(1234) + + +def test_validate_aiperf_version_rejects_uncovered_release(tmp_path) -> None: + fake = tmp_path / "aiperf" + fake.write_text("#!/bin/sh\nprintf '0.12.0\\n'\n") + fake.chmod(0o755) + + with pytest.raises(RuntimeError, match="AIPerf 0.12.0 is unsupported"): + validate_aiperf_version(str(fake)) + + +def test_aggregate_exports_combines_independent_runs(tmp_path) -> None: + exports = [] + for index, throughput in enumerate((10.0, 14.0), start=1): + path = tmp_path / f"run-{index}.json" + document = { + "aiperf_version": "0.11.0", + "error_request_count": None, + "request_count": {"unit": "requests", "avg": 20}, + "request_throughput": { + "unit": "requests/sec", + "avg": throughput, + }, + "time_to_first_token": {"unit": "ms", "p50": 10.0}, + } + if index == 2: + del document["error_request_count"] + document["error_summary"] = [] + path.write_text(json.dumps(document)) + exports.append(path) + + aggregate_path = aggregate_exports(exports, tmp_path / "aggregate" / "report.json") + metrics = json.loads(aggregate_path.read_text())["metrics"] + + assert metrics["request_throughput_avg"]["mean"] == 12.0 + assert metrics["request_throughput_avg"]["ci_low"] < 0 + assert metrics["error_request_count_avg"]["mean"] == 0.0 + + mismatched = json.loads(exports[1].read_text()) + mismatched["time_to_first_token"]["unit"] = "seconds" + exports[1].write_text(json.dumps(mismatched)) + with pytest.raises(RuntimeError, match="disagree on the unit"): + aggregate_exports(exports, tmp_path / "invalid" / "report.json") + + mismatched["time_to_first_token"]["unit"] = "ms" + mismatched["request_count"]["avg"] = True + exports[1].write_text(json.dumps(mismatched)) + with pytest.raises(RuntimeError, match="has no numeric request_count.avg"): + aggregate_exports(exports, tmp_path / "malformed" / "report.json") diff --git a/tests/test_routing_overhead_plot.py b/tests/test_routing_overhead_plot.py new file mode 100644 index 000000000..9a7ddf42e --- /dev/null +++ b/tests/test_routing_overhead_plot.py @@ -0,0 +1,64 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import xml.etree.ElementTree as ET + +from scripts.routing_overhead_plot import OverheadPlotRow, write_overhead_plot + + +def test_overhead_plot_preserves_matrix_and_unavailable_values(tmp_path) -> None: + path = tmp_path / "routing-overhead.svg" + rows = ( + OverheadPlotRow("short-", "fixed", "random", 3.0, 25.0, -80.0, -11.43), + OverheadPlotRow("short-", "fixed", "stage&router", None, None, -14.0, -2.0), + OverheadPlotRow("long-context", "traffic-burst", "random", 1.25, 3.0, None, None), + OverheadPlotRow("long-context", "traffic-burst", "stage&router", -0.5, -1.0, 7.0, 1.0), + ) + + write_overhead_plot(path, rows) + + root = ET.fromstring(path.read_text()) + namespace = {"svg": "http://www.w3.org/2000/svg"} + cells = root.findall(".//svg:g", namespace) + text_nodes = root.findall(".//svg:text", namespace) + values = { + ( + cell.attrib["data-scenario"], + cell.attrib["data-load"], + cell.attrib["data-route"], + cell.attrib["data-metric"], + ): " ".join("".join(cell.itertext()).split()) + for cell in cells + } + assert len(values) == 8 + assert ( + values[ + ( + "short-", + "fixed", + "random", + "output_tokens_per_second_delta", + ) + ] + == "short-, fixed, random: -80.00 tokens/s (-11.43%) " + "-80.00 tokens/s (-11.43%)" + ) + assert ( + "n/a" + in values[("long-context", "traffic-burst", "random", "output_tokens_per_second_delta")] + ) + assert "n/a" in values[("short-", "fixed", "stage&router", "ttft_p50_delta_ms")] + centered_labels = { + node.text: node.attrib["text-anchor"] + for node in text_nodes + if node.text in {"Routing overhead versus the direct backend", "short "} + } + assert centered_labels == { + "Routing overhead versus the direct backend": "middle", + "short ": "middle", + } + assert all( + text.attrib["text-anchor"] == "middle" + for cell in cells + for text in cell.findall("svg:text", namespace) + ) diff --git a/tests/test_routing_performance_report.py b/tests/test_routing_performance_report.py new file mode 100644 index 000000000..445357cc1 --- /dev/null +++ b/tests/test_routing_performance_report.py @@ -0,0 +1,391 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import csv +import json +from dataclasses import replace +from pathlib import Path + +import pytest + +import scripts.benchmark_routing_algorithms as benchmark +from scripts.benchmark_routing_algorithms import ( + BenchmarkArm, + BenchmarkConfig, + DirectBaseline, + ScenarioDefinition, + compare_to_direct_backend, + materialize_aiperf_input, + parse_result, + profile_request_count, + write_report, +) + + +def test_report_combines_scenarios_single_and_aggregate_results(tmp_path) -> None: + oha_path = tmp_path / "oha.json" + single_path = tmp_path / "profile_export_aiperf.json" + direct_path = tmp_path / "profile_export_aiperf_direct.json" + aggregate_path = tmp_path / "profile_export_aiperf_aggregate.json" + oha_path.write_text( + json.dumps( + { + "summary": {"successRate": 1.0, "requestsPerSec": 120.5}, + "latencyPercentiles": {"p50": 0.012, "p99": 0.045}, + } + ) + ) + single_path.write_text( + json.dumps( + { + "error_request_count": {"unit": "requests", "avg": 0}, + "request_count": {"unit": "requests", "avg": 4}, + "request_throughput": {"unit": "requests/sec", "avg": 95.5}, + "request_latency": {"unit": "ms", "p50": 40.0}, + "time_to_first_token": {"unit": "ms", "p50": 15.0, "p99": 35.0}, + "output_token_throughput": {"unit": "tokens/sec", "avg": 620.0}, + "e2e_output_token_throughput": { + "unit": "tokens/sec/user", + "p50": 200.0, + }, + } + ) + ) + direct_path.write_text( + json.dumps( + { + "error_request_count": {"unit": "requests", "avg": 0}, + "request_count": {"unit": "requests", "avg": 4}, + "request_throughput": {"unit": "requests/sec", "avg": 100.0}, + "request_latency": {"unit": "ms", "p50": 35.0}, + "time_to_first_token": {"unit": "ms", "p50": 12.0, "p99": 30.0}, + "output_token_throughput": {"unit": "tokens/sec", "avg": 700.0}, + } + ) + ) + aggregate_path.write_text( + json.dumps( + { + "metrics": { + "error_request_count_avg": {"unit": "requests", "mean": 2}, + "request_count_avg": {"unit": "requests", "mean": 4}, + "request_throughput_avg": { + "unit": "requests/sec", + "mean": 80.0, + "cv": 0.03, + "ci_low": 77.0, + "ci_high": 83.0, + }, + "request_latency_p50": {"unit": "seconds", "mean": 0.05}, + "time_to_first_token_p50": {"unit": "ms", "mean": 20.0}, + "time_to_first_token_p99": {"unit": "ms", "mean": 45.0}, + "output_token_throughput_avg": { + "unit": "tokens/sec", + "mean": 500.0, + }, + } + } + ) + ) + short = ScenarioDefinition( + id="short-interactive", + group="core", + description="short baseline", + expected="all requests succeed", + expected_error_rate_min=0.0, + expected_error_rate_max=0.0, + input_file=tmp_path / "short.json", + load_profiles=(), + ) + failure = ScenarioDefinition( + id="failure-pressure", + group="resilience", + description="failure injection", + expected="failures stay bounded", + expected_error_rate_min=0.01, + expected_error_rate_max=0.75, + input_file=tmp_path / "failure.json", + load_profiles=(), + ) + before = { + "models": { + "mock/weak": {"calls": 10, "errors": 0}, + "mock/classifier": {"calls": 3, "errors": 0}, + }, + "classifier": { + "total_requests": 3, + "total_errors": 0, + "models": { + "mock/classifier": { + "calls": 3, + "errors": 0, + "model_call_latency": {"count": 3, "total_ms": 6}, + } + }, + }, + "routing_overhead": {"count": 10, "total_ms": 20}, + } + after = { + "models": { + "mock/weak": {"calls": 20, "errors": 2}, + "mock/strong": {"calls": 2, "errors": 0}, + "mock/classifier": {"calls": 5, "errors": 1}, + }, + "classifier": { + "total_requests": 5, + "total_errors": 1, + "models": { + "mock/classifier": { + "calls": 5, + "errors": 1, + "model_call_latency": {"count": 5, "total_ms": 14}, + } + }, + }, + "routing_overhead": {"count": 22, "total_ms": 50}, + } + results = compare_to_direct_backend( + ( + parse_result( + BenchmarkArm( + "direct-backend", + "mock/weak", + "http://127.0.0.1:8100", + True, + ), + short, + "fixed", + None, + direct_path, + {}, + {}, + ), + parse_result( + BenchmarkArm( + "random", + "switchyard/random", + "http://127.0.0.1:4000", + False, + ), + short, + "fixed", + oha_path, + single_path, + before, + after, + ), + parse_result( + BenchmarkArm( + "classifier", + "switchyard/classifier", + "http://127.0.0.1:4000", + False, + ), + failure, + "fixed", + None, + aggregate_path, + before, + after, + ), + ) + ) + output_dir = tmp_path / "report" + output_dir.mkdir() + config = BenchmarkConfig( + base_url="http://127.0.0.1:4000", + models=(("random", "switchyard/random"),), + concurrency=100, + request_count=1000, + backend_label="test backend", + output_dir=output_dir, + oha_bin="oha", + aiperf_bin="aiperf", + soak_bin="switchyard-soak", + direct_baseline=DirectBaseline("http://127.0.0.1:8100", "mock/weak"), + ) + + write_report(config, results) + + report = (output_dir / "report.md").read_text() + assert "## Routing overhead versus the direct backend" in report + assert "![Routing overhead plots](routing-overhead.svg)" in report + assert ( + report.index("## Routing overhead versus the direct backend") + < report.index("## Run details") + < report.index("## Throughput and latency") + ) + assert ( + "| short interactive | random | fixed | +5.00 ms | +3.00 ms | +25.00% | " + "-4.50% | -80.00 tokens/s | -11.43% |" in report + ) + assert "| short interactive | short baseline |" in report + assert "| random | short-interactive | fixed | 120.50 | 95.50 |" in report + assert "## Repeatability" in report + assert "## Resilience" in report + assert ( + "| classifier | failure-pressure | fixed | 50.00% | 1.00%–75.00% | PASS | `{" + '"mock/weak":2}` | failures stay bounded |' in report + ) + rows = list(csv.DictReader((output_dir / "report.csv").open())) + rows_by_key = {(row["algorithm"], row["scenario"]): row for row in rows} + random_row = rows_by_key[("random", "short-interactive")] + assert random_row["selected_model_calls"] == '{"mock/strong":2,"mock/weak":10}' + assert random_row["selected_model_share"] == '{"mock/strong":0.1667,"mock/weak":0.8333}' + assert random_row["selected_model_errors"] == '{"mock/weak":2}' + payload = json.loads((output_dir / "report.json").read_text()) + payload_by_key = {(row["algorithm"], row["scenario"]): row for row in payload["results"]} + direct_row = payload_by_key[("direct-backend", "short-interactive")] + random_payload = payload_by_key[("random", "short-interactive")] + classifier_payload = payload_by_key[("classifier", "failure-pressure")] + assert direct_row["aiperf_itl_p50_ms"] is None + assert direct_row["aiperf_request_throughput_delta_pct"] is None + assert random_payload["aiperf_output_tokens_per_second_per_user_p50"] == 200.0 + assert random_payload["aiperf_ttft_p50_delta_pct"] == 25.0 + assert random_payload["aiperf_output_tokens_per_second_delta"] == -80.0 + assert random_payload["scenario_description"] == "short baseline" + assert classifier_payload["aiperf_request_latency_p50_ms"] == 50.0 + assert classifier_payload["aiperf_request_throughput_cv"] == 0.03 + assert classifier_payload["classifier_latency_avg_ms"] == 4.0 + + assert (output_dir / "routing-overhead.svg").is_file() + + with pytest.raises(RuntimeError, match="missing direct baseline for mixed-traffic fixed"): + compare_to_direct_backend((results[0], replace(results[1], scenario="mixed-traffic"))) + + +def test_materialized_sessions_are_unique_and_cover_request_count(tmp_path) -> None: + scenario_path = tmp_path / "short.json" + scenario_path.write_text( + json.dumps( + { + "data": [ + { + "session_id": "short-0", + "payloads": [{"model": "test"}, {"model": "test"}], + } + ] + } + ) + ) + scenario = ScenarioDefinition( + id="short-interactive", + group="core", + description="short baseline", + expected="all requests succeed", + expected_error_rate_min=0.0, + expected_error_rate_max=0.0, + input_file=scenario_path, + load_profiles=(), + ) + + expanded_input = materialize_aiperf_input(scenario, tmp_path / "expanded.json", 5) + expanded_sessions = json.loads(expanded_input.read_text())["data"] + + assert len(expanded_sessions) == 3 + assert len({session["session_id"] for session in expanded_sessions}) == 3 + + +def test_traffic_burst_request_count_integrates_rate_series(tmp_path) -> None: + config = BenchmarkConfig( + base_url="http://127.0.0.1:4000", + models=(("random", "switchyard/random"),), + concurrency=2, + request_count=100, + backend_label="test backend", + output_dir=tmp_path, + oha_bin="oha", + aiperf_bin="aiperf", + soak_bin="switchyard-soak", + ) + profile = { + "kind": "traffic_burst", + "duration_seconds": 30, + "points": [ + {"time_s": 0, "rate_multiplier": 1}, + {"time_s": 10, "rate_multiplier": 1}, + {"time_s": 11, "rate_multiplier": 10}, + {"time_s": 16, "rate_multiplier": 10}, + {"time_s": 17, "rate_multiplier": 1}, + {"time_s": 30, "rate_multiplier": 1}, + ], + } + + assert profile_request_count(config, profile) == 168 + + +def test_aiperf_cells_use_disjoint_artifacts(tmp_path, monkeypatch) -> None: + scenario_path = tmp_path / "short.json" + scenario_path.write_text( + json.dumps( + { + "data": [ + { + "session_id": "short-0", + "payloads": [{"model": "switchyard/random"}], + } + ] + } + ) + ) + scenario = ScenarioDefinition( + id="short-interactive", + group="core", + description="short baseline", + expected="all requests succeed", + expected_error_rate_min=0.0, + expected_error_rate_max=0.0, + input_file=scenario_path, + load_profiles=(), + ) + config = BenchmarkConfig( + base_url="http://127.0.0.1:4000", + models=(("random", "switchyard/random"),), + concurrency=1, + request_count=1, + backend_label="test backend", + output_dir=tmp_path, + oha_bin="oha", + aiperf_bin="aiperf", + soak_bin="switchyard-soak", + profile_runs=1, + ) + arm = BenchmarkArm( + "random", + "switchyard/random", + "http://127.0.0.1:4000", + False, + ) + observed: list[tuple[Path, Path]] = [] + + def fake_run_profile( + _command, log_path: Path, artifact_dir: Path, _timeout_seconds: int + ) -> Path: + observed.append((log_path, artifact_dir)) + artifact_dir.mkdir(parents=True) + export = artifact_dir / "profile_export_aiperf.json" + export.write_text("{}") + return export + + monkeypatch.setattr(benchmark, "run_profile", fake_run_profile) + + first = benchmark.run_aiperf( + config, + arm, + scenario, + {"kind": "fixed"}, + tmp_path, + "algorithm-00-short-interactive-fixed", + ) + second = benchmark.run_aiperf( + config, + arm, + scenario, + {"kind": "fixed"}, + tmp_path, + "algorithm-00-long-context-fixed", + ) + + assert first != second + assert observed[0][0] != observed[1][0] + assert observed[0][1] != observed[1][1]