From c8a1632a697c92043528a7b22bfd63cc1652d39c Mon Sep 17 00:00:00 2001 From: Elyas Mehtabuddin Date: Thu, 30 Jul 2026 09:45:49 -0700 Subject: [PATCH 1/9] feat(operations): add release soak test Signed-off-by: Elyas Mehtabuddin --- .gitignore | 1 + Cargo.lock | 15 + Cargo.toml | 1 + crates/switchyard-soak/Cargo.toml | 30 ++ crates/switchyard-soak/src/client.rs | 364 +++++++++++++++++ crates/switchyard-soak/src/lib.rs | 564 +++++++++++++++++++++++++++ crates/switchyard-soak/src/main.rs | 10 + crates/switchyard-soak/src/report.rs | 355 +++++++++++++++++ crates/switchyard-soak/src/stats.rs | 560 ++++++++++++++++++++++++++ crates/switchyard-soak/tests/soak.rs | 318 +++++++++++++++ docs/operations/soak_test.md | 194 +++++++++ mkdocs.yml | 1 + scripts/soak-rehearsal.sh | 87 +++++ 13 files changed, 2500 insertions(+) create mode 100644 crates/switchyard-soak/Cargo.toml create mode 100644 crates/switchyard-soak/src/client.rs create mode 100644 crates/switchyard-soak/src/lib.rs create mode 100644 crates/switchyard-soak/src/main.rs create mode 100644 crates/switchyard-soak/src/report.rs create mode 100644 crates/switchyard-soak/src/stats.rs create mode 100644 crates/switchyard-soak/tests/soak.rs create mode 100644 docs/operations/soak_test.md create mode 100755 scripts/soak-rehearsal.sh diff --git a/.gitignore b/.gitignore index fdc44128c..ec398cb0b 100644 --- a/.gitignore +++ b/.gitignore @@ -150,6 +150,7 @@ plans/ # Benchmark result directories bench_results*/ +soak-results/ jobs/**/*.* pytest-of-*/ diff --git a/Cargo.lock b/Cargo.lock index 405dda9da..41f24d376 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2388,6 +2388,21 @@ dependencies = [ "tokio", ] +[[package]] +name = "switchyard-soak" +version = "0.2.0" +dependencies = [ + "axum", + "clap", + "futures-util", + "parking_lot", + "rand 0.10.2", + "reqwest", + "serde_json", + "tempfile", + "tokio", +] + [[package]] name = "switchyard-translation" version = "0.2.0" diff --git a/Cargo.toml b/Cargo.toml index 98433d8c1..d929d0c58 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,6 +10,7 @@ members = [ "crates/protocol", "crates/switchyard-server", "crates/switchyard-skill-distillation", + "crates/switchyard-soak", "crates/switchyard-translation", ] diff --git a/crates/switchyard-soak/Cargo.toml b/crates/switchyard-soak/Cargo.toml new file mode 100644 index 000000000..771cd78be --- /dev/null +++ b/crates/switchyard-soak/Cargo.toml @@ -0,0 +1,30 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "switchyard-soak" +version.workspace = true +description = "Sustained load tester for a live Switchyard server" +authors.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true + +[[bin]] +name = "switchyard-soak" +path = "src/main.rs" + +[dependencies] +clap = { version = "4", features = ["derive"] } +futures-util.workspace = true +parking_lot.workspace = true +rand.workspace = true +reqwest.workspace = true +serde_json.workspace = true +tokio.workspace = true + +[dev-dependencies] +axum = "0.8" +tempfile = "3" +tokio.workspace = true diff --git a/crates/switchyard-soak/src/client.rs b/crates/switchyard-soak/src/client.rs new file mode 100644 index 000000000..e3f3285f5 --- /dev/null +++ b/crates/switchyard-soak/src/client.rs @@ -0,0 +1,364 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! HTTP against the public Switchyard APIs: preflight, one request, and server-state reads. + +use std::collections::BTreeMap; + +use clap::ValueEnum; +use futures_util::StreamExt; +use reqwest::Client; +use reqwest::header::CONTENT_TYPE; +use serde_json::{Value, json}; + +use crate::stats::{SERVER_ERRORS_METRIC, SERVER_REQUESTS_METRIC}; + +/// One public Switchyard API the soak test exercises. +#[derive(Clone, Copy, PartialEq, Eq, ValueEnum)] +pub enum Endpoint { + Chat, + Messages, + Responses, +} + +impl Endpoint { + pub const ALL: [Endpoint; 3] = [Endpoint::Chat, Endpoint::Messages, Endpoint::Responses]; + + pub fn path(self) -> &'static str { + match self { + Endpoint::Chat => "/v1/chat/completions", + Endpoint::Messages => "/v1/messages", + Endpoint::Responses => "/v1/responses", + } + } + + /// Field a successful response for this endpoint must contain. + fn required_field(self) -> &'static str { + match self { + Endpoint::Chat => "choices", + Endpoint::Messages => "content", + Endpoint::Responses => "output", + } + } + + pub fn as_str(self) -> &'static str { + match self { + Endpoint::Chat => "chat", + Endpoint::Messages => "messages", + Endpoint::Responses => "responses", + } + } +} + +/// Keep at most *limit* characters, on a char boundary, for a logged detail string. +fn truncate(text: &str, limit: usize) -> String { + text.chars().take(limit).collect() +} + +/// Map a transport-level reqwest error to a short, stable error kind. +fn transport_error_kind(error: &reqwest::Error) -> &'static str { + if error.is_timeout() { + "timeout" + } else if error.is_decode() { + // A body decode failure mirrors httpx's DecodingError, which reported "request_error". + "request_error" + } else if error.is_connect() || error.is_request() || error.is_body() { + // Connect, send, and mid-stream read failures are httpx TransportError -> "transport". + "transport" + } else { + "request_error" + } +} + +/// Build one request body for a public Switchyard API. +pub fn request_body( + endpoint: Endpoint, + model: &str, + prompt: &str, + max_output_tokens: u32, + stream: bool, +) -> Value { + match endpoint { + // Chat Completions and Anthropic Messages take the same model/messages/max_tokens body. + Endpoint::Chat | Endpoint::Messages => json!({ + "model": model, + "messages": [{"role": "user", "content": prompt}], + "max_tokens": max_output_tokens, + "temperature": 0, + "stream": stream, + }), + Endpoint::Responses => json!({ + "model": model, + "input": prompt, + "max_output_tokens": max_output_tokens, + "stream": stream, + }), + } +} + +/// Read the process-wide Switchyard counters from Prometheus text. +pub fn parse_metrics(text: &str) -> BTreeMap { + let wanted = [SERVER_REQUESTS_METRIC, SERVER_ERRORS_METRIC]; + let mut parsed = BTreeMap::new(); + for line in text.lines() { + if line.is_empty() || line.starts_with('#') { + continue; + } + let Some((name_part, rest)) = line.split_once(' ') else { + continue; + }; + let name = name_part.split('{').next().unwrap_or(name_part); + if !wanted.contains(&name) { + continue; + } + if let Some(token) = rest.split_whitespace().next() + && let Ok(value) = token.parse::() + { + parsed.insert(name.to_string(), value); + } + } + parsed +} + +/// Send one request and return an error kind and detail, if any. +pub async fn send_request( + client: &Client, + base_url: &str, + endpoint: Endpoint, + body: &Value, +) -> (Option, Option) { + let url = format!("{base_url}{}", endpoint.path()); + let stream = body.get("stream").and_then(Value::as_bool).unwrap_or(false); + let response = match client.post(&url).json(body).send().await { + Ok(response) => response, + Err(error) => { + return ( + Some(transport_error_kind(&error).to_string()), + Some(truncate(&error.to_string(), 500)), + ); + } + }; + + if stream { + let status = response.status(); + if !status.is_success() { + let content = response.text().await.unwrap_or_default(); + return ( + Some(format!("http_{}", status.as_u16())), + Some(truncate(&content, 500)), + ); + } + let content_type = response + .headers() + .get(CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .unwrap_or("") + .to_string(); + if !content_type.contains("text/event-stream") { + let content = response.text().await.unwrap_or_default(); + return ( + Some("invalid_stream".to_string()), + Some(format!( + "expected text/event-stream, received {content_type:?}: {}", + truncate(&content, 300) + )), + ); + } + let mut bytes = response.bytes_stream(); + let mut received_data = false; + while let Some(chunk) = bytes.next().await { + match chunk { + Ok(chunk) => { + received_data = + received_data || chunk.iter().any(|byte| !byte.is_ascii_whitespace()); + } + Err(error) => { + return ( + Some(transport_error_kind(&error).to_string()), + Some(truncate(&error.to_string(), 500)), + ); + } + } + } + if !received_data { + return ( + Some("empty_stream".to_string()), + Some("successful streaming response contained no data".to_string()), + ); + } + return (None, None); + } + + let status = response.status(); + if !status.is_success() { + let content = response.text().await.unwrap_or_default(); + return ( + Some(format!("http_{}", status.as_u16())), + Some(truncate(&content, 500)), + ); + } + let text = match response.text().await { + Ok(text) => text, + Err(error) => { + return ( + Some(transport_error_kind(&error).to_string()), + Some(truncate(&error.to_string(), 500)), + ); + } + }; + let payload: Value = match serde_json::from_str(&text) { + Ok(payload) => payload, + Err(error) => return (Some("invalid_json".to_string()), Some(error.to_string())), + }; + if !payload.is_object() || payload.get("error").is_some() { + return ( + Some("invalid_response".to_string()), + Some(truncate(&text, 500)), + ); + } + let field = endpoint.required_field(); + if payload.get(field).is_none() { + return ( + Some("invalid_response".to_string()), + Some(format!( + "successful {} response did not contain {field:?}: {}", + endpoint.as_str(), + truncate(&text, 300) + )), + ); + } + (None, None) +} + +/// Check liveness and model discovery, then return the selected model. +pub async fn preflight( + client: &Client, + base_url: &str, + requested_model: Option<&str>, +) -> Result { + let health = client + .get(format!("{base_url}/health")) + .send() + .await + .map_err(|error| error.to_string())? + .error_for_status() + .map_err(|error| error.to_string())?; + let health_text = health.text().await.map_err(|error| error.to_string())?; + let health_body: Value = serde_json::from_str(&health_text).map_err(|_| { + format!( + "GET /health did not return JSON: {}", + truncate(&health_text, 300) + ) + })?; + if health_body.get("status").and_then(Value::as_str) != Some("ok") { + return Err(format!( + "GET /health returned an unexpected body: {}", + truncate(&health_text, 300) + )); + } + + let response = client + .get(format!("{base_url}/v1/models")) + .send() + .await + .map_err(|error| error.to_string())? + .error_for_status() + .map_err(|error| error.to_string())?; + let text = response.text().await.map_err(|error| error.to_string())?; + let body: Value = serde_json::from_str(&text).map_err(|_| { + format!( + "GET /v1/models did not return JSON: {}", + truncate(&text, 300) + ) + })?; + let entries = body.get("data").and_then(Value::as_array).ok_or_else(|| { + format!( + "GET /v1/models returned an unexpected body: {}", + truncate(&text, 300) + ) + })?; + let model_ids: Vec = entries + .iter() + .filter_map(|entry| entry.get("id").and_then(Value::as_str).map(str::to_string)) + .collect(); + let default_model = body + .get("default_model") + .and_then(Value::as_str) + .filter(|id| model_ids.iter().any(|known| known == id)); + + let model = requested_model + .map(str::to_string) + .or_else(|| default_model.map(str::to_string)) + .or_else(|| model_ids.first().cloned()) + .ok_or_else(|| "GET /v1/models returned no model to test".to_string())?; + if let Some(requested) = requested_model + && !model_ids.iter().any(|known| known == requested) + { + return Err(format!( + "model {requested:?} is not listed by GET /v1/models" + )); + } + Ok(model) +} + +/// Read liveness and cumulative server metrics; never errors, so one bad read is one failed sample. +pub async fn read_server_state(client: &Client, base_url: &str) -> (bool, BTreeMap) { + let healthy = match client.get(format!("{base_url}/health")).send().await { + Ok(response) if response.status() == reqwest::StatusCode::OK => response + .text() + .await + .ok() + .and_then(|text| serde_json::from_str::(&text).ok()) + .and_then(|body| { + body.get("status") + .and_then(Value::as_str) + .map(|s| s == "ok") + }) + .unwrap_or(false), + _ => false, + }; + let metrics = match client.get(format!("{base_url}/metrics")).send().await { + Ok(response) if response.status().is_success() => response + .text() + .await + .map(|text| parse_metrics(&text)) + .unwrap_or_default(), + _ => BTreeMap::new(), + }; + (healthy, metrics) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn request_bodies_match_public_endpoints() { + let chat = request_body(Endpoint::Chat, "route", "hello", 8, true); + let messages = request_body(Endpoint::Messages, "route", "hello", 8, true); + let responses = request_body(Endpoint::Responses, "route", "hello", 8, true); + + assert_eq!(chat["messages"][0]["content"], json!("hello")); + assert_eq!(chat["max_tokens"], json!(8)); + assert_eq!(messages["max_tokens"], json!(8)); + assert_eq!(responses["input"], json!("hello")); + assert_eq!(responses["max_output_tokens"], json!(8)); + for body in [&chat, &messages, &responses] { + assert_eq!(body["stream"], json!(true)); + } + } + + #[test] + fn parse_metrics_reads_counter_names_and_ignores_labelled_series() { + let metrics = parse_metrics( + "# TYPE switchyard_total_requests gauge\n\ + switchyard_total_requests 42\n\ + switchyard_total_errors{} 3\n\ + switchyard_requests_total{model=\"route\"} 10\n", + ); + + assert_eq!(metrics.get("switchyard_total_requests"), Some(&42.0)); + assert_eq!(metrics.get("switchyard_total_errors"), Some(&3.0)); + assert_eq!(metrics.get("switchyard_requests_total"), None); + } +} diff --git a/crates/switchyard-soak/src/lib.rs b/crates/switchyard-soak/src/lib.rs new file mode 100644 index 000000000..2dbfe1db8 --- /dev/null +++ b/crates/switchyard-soak/src/lib.rs @@ -0,0 +1,564 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Sustained, closed-loop load test against a live Switchyard server. +//! +//! Workers keep a fixed number of inference requests in flight while a reporter samples +//! liveness, metrics, and process resources, and a canary confirms invalid input is rejected. +//! The run writes result files and exits non-zero when a release gate fails. + +pub mod client; +pub mod report; +pub mod stats; + +use std::fs; +use std::future::Future; +use std::path::{Path, PathBuf}; +use std::process::ExitCode; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::time::{Duration, Instant}; + +use clap::Parser; +use parking_lot::Mutex; +use rand::rngs::StdRng; +use rand::{RngExt, SeedableRng}; +use reqwest::Client; +use reqwest::header::{AUTHORIZATION, HeaderMap, HeaderValue}; +use serde_json::json; +use tokio::sync::Notify; + +use crate::client::{Endpoint, preflight, request_body, send_request}; +use crate::report::{ResultsWriter, invalid_request_canary, reporter}; +use crate::stats::{ + RunStats, build_prompt_pool, build_summary, latency_report, now_utc_string, round3, + utc_dir_stamp, +}; + +/// Command-line arguments for the soak test. +#[derive(Parser)] +#[command( + name = "switchyard-soak", + about = "Run a sustained, closed-loop load test against a live Switchyard server", + version +)] +pub struct Args { + /// Base URL of the Switchyard server. + #[arg(long, default_value = "http://127.0.0.1:4000")] + base_url: String, + + /// Model id from GET /v1/models; defaults to its first model. + #[arg(long)] + model: Option, + + /// Run time with an s, m, or h suffix. + #[arg(long, value_parser = stats::parse_duration, default_value = "48h")] + duration: f64, + + /// Number of closed-loop inference workers. + #[arg(long, default_value_t = 16)] + concurrency: usize, + + /// Public APIs to exercise; defaults to all three. + #[arg(long, value_enum, num_args = 1..)] + endpoints: Vec, + + /// Fraction of inference requests that use streaming. + #[arg(long, default_value_t = 0.5)] + stream_ratio: f64, + + /// Maximum output tokens requested from the backend. + #[arg(long, default_value_t = 32)] + max_output_tokens: u32, + + /// Repeated-prefix payload size for each prompt. + #[arg(long, default_value_t = 1024)] + prompt_bytes: usize, + + /// Timeout in seconds for one inference request. + #[arg(long, default_value_t = 120.0)] + request_timeout: f64, + + /// Seconds between health, metrics, and result samples. + #[arg(long, default_value_t = 60.0)] + report_interval: f64, + + /// Seconds between invalid-request recovery checks; zero disables them. + #[arg(long, default_value_t = 300.0)] + invalid_canary_interval: f64, + + /// Largest allowed inference error fraction. + #[arg(long, default_value_t = 0.0)] + max_error_rate: f64, + + /// Local Switchyard PID to sample for RSS and CPU. + #[arg(long)] + server_pid: Option, + + /// Largest allowed first-to-last RSS increase in MiB. + #[arg(long, requires = "server_pid")] + max_rss_growth_mib: Option, + + /// Environment variable holding a bearer token for the Switchyard endpoint. + #[arg(long)] + api_key_env: Option, + + /// New directory for the run result files. + #[arg(long)] + results_dir: Option, +} + +impl Args { + /// Reject inputs clap's types cannot, matching the Python runner's checks. + pub fn validate(&self) -> Result<(), String> { + if self.concurrency == 0 { + return Err("--concurrency must be greater than zero".to_string()); + } + let mut seen = std::collections::HashSet::new(); + if !self + .endpoints + .iter() + .all(|endpoint| seen.insert(endpoint.as_str())) + { + return Err("--endpoints must not repeat a value".to_string()); + } + if !(0.0..=1.0).contains(&self.stream_ratio) { + return Err("--stream-ratio must be between 0 and 1".to_string()); + } + if self.max_output_tokens == 0 || self.prompt_bytes == 0 { + return Err( + "--max-output-tokens and --prompt-bytes must be greater than zero".to_string(), + ); + } + if self.request_timeout <= 0.0 || self.report_interval <= 0.0 { + return Err( + "--request-timeout and --report-interval must be greater than zero".to_string(), + ); + } + if self.invalid_canary_interval < 0.0 { + return Err("--invalid-canary-interval must be zero or greater".to_string()); + } + if !(0.0..=1.0).contains(&self.max_error_rate) { + return Err("--max-error-rate must be between 0 and 1".to_string()); + } + if self.server_pid == Some(0) { + return Err("--server-pid must be greater than zero".to_string()); + } + if self.max_rss_growth_mib.is_some_and(|growth| growth < 0.0) { + return Err("--max-rss-growth-mib must be zero or greater".to_string()); + } + Ok(()) + } + + fn endpoints(&self) -> Vec { + if self.endpoints.is_empty() { + Endpoint::ALL.to_vec() + } else { + self.endpoints.clone() + } + } +} + +/// A one-shot stop signal that many tasks can wait on and any task can raise. +pub struct Stop { + flag: AtomicBool, + notify: Notify, +} + +impl Stop { + pub fn new() -> Self { + Self { + flag: AtomicBool::new(false), + notify: Notify::new(), + } + } + + pub fn set(&self) { + self.flag.store(true, Ordering::SeqCst); + self.notify.notify_waiters(); + } + + pub fn is_set(&self) -> bool { + self.flag.load(Ordering::SeqCst) + } + + /// Resolve once the signal is raised, now or later. + pub async fn wait(&self) { + loop { + if self.is_set() { + return; + } + // Register before the second flag check so a set() between the two still wakes us. + let notified = self.notify.notified(); + tokio::pin!(notified); + notified.as_mut().enable(); + if self.is_set() { + return; + } + notified.await; + } + } +} + +impl Default for Stop { + fn default() -> Self { + Self::new() + } +} + +/// Send closed-loop traffic until the run stops. +#[allow(clippy::too_many_arguments)] +async fn worker( + client: Client, + base_url: String, + worker_id: usize, + model: String, + endpoints: Vec, + prompt_pool: Vec, + max_output_tokens: u32, + stream_ratio: f64, + stop: Arc, + request_numbers: Arc, + stats: Arc>, + writer: Arc>, +) -> Result<(), String> { + let mut rng = StdRng::seed_from_u64(10_000 + worker_id as u64); + while !stop.is_set() { + let request_number = request_numbers.fetch_add(1, Ordering::Relaxed) as usize; + let endpoint = endpoints[request_number % endpoints.len()]; + let stream = rng.random::() < stream_ratio; + let body = request_body( + endpoint, + &model, + &prompt_pool[request_number % prompt_pool.len()], + max_output_tokens, + stream, + ); + let started = Instant::now(); + let (error_kind, detail) = send_request(&client, &base_url, endpoint, &body).await; + let latency_ms = started.elapsed().as_secs_f64() * 1000.0; + stats + .lock() + .record(endpoint.as_str(), latency_ms, error_kind.as_deref()); + if let Some(kind) = &error_kind { + writer + .lock() + .write_error(&json!({ + "timestamp_utc": now_utc_string(), + "worker": worker_id, + "endpoint": endpoint.as_str(), + "stream": stream, + "latency_ms": round3(latency_ms), + "error": kind, + "detail": detail, + })) + .map_err(|error| error.to_string())?; + } + } + Ok(()) +} + +/// Run a background task and raise the stop signal unless it returns `Ok`, so the run ends +/// fail-closed. The drop guard fires on an `Err` return and on a panic unwinding through the +/// await, matching the Python runner, which stopped the run on any task exception; a task that +/// returns `Ok` (a worker/reporter after stop, or a disabled canary) leaves the guard disarmed. +async fn guard_stop( + stop: Arc, + task: impl Future>, +) -> Result<(), String> { + struct StopOnDrop(Arc, bool); + impl Drop for StopOnDrop { + fn drop(&mut self) { + if self.1 { + self.0.set(); + } + } + } + let mut guard = StopOnDrop(stop, true); + let result = task.await; + guard.1 = result.is_err(); + result +} + +/// Fold one joined task's outcome into the failure reasons; cancelled tasks are expected. +fn collect_failure( + name: &str, + result: Result, tokio::task::JoinError>, + failures: &mut Vec, +) { + match result { + Ok(Ok(())) => {} + Ok(Err(reason)) => failures.push(format!("{name} failed: {reason}")), + Err(join) if join.is_cancelled() => {} + Err(join) => failures.push(format!("{name} failed: {join}")), + } +} + +/// Raise *stop* on SIGINT or SIGTERM so an operator can end the run cleanly. +fn spawn_signal_listener(stop: Arc) { + tokio::spawn(async move { + #[cfg(unix)] + { + use tokio::signal::unix::{SignalKind, signal}; + let mut interrupt = signal(SignalKind::interrupt()).ok(); + let mut terminate = signal(SignalKind::terminate()).ok(); + let wait_interrupt = async { + match interrupt.as_mut() { + Some(stream) => { + stream.recv().await; + } + None => std::future::pending::<()>().await, + } + }; + let wait_terminate = async { + match terminate.as_mut() { + Some(stream) => { + stream.recv().await; + } + None => std::future::pending::<()>().await, + } + }; + tokio::select! { + _ = wait_interrupt => {} + _ = wait_terminate => {} + } + stop.set(); + } + #[cfg(not(unix))] + { + let _ = tokio::signal::ctrl_c().await; + stop.set(); + } + }); +} + +/// Record every non-secret input, plus the resolved model and the seconds duration. +fn write_config(results_dir: &Path, args: &Args, model: &str) -> Result<(), String> { + let config = json!({ + "base_url": args.base_url, + "model": model, + "duration_seconds": args.duration, + "concurrency": args.concurrency, + "endpoints": args.endpoints().iter().map(|endpoint| endpoint.as_str()).collect::>(), + "stream_ratio": args.stream_ratio, + "max_output_tokens": args.max_output_tokens, + "prompt_bytes": args.prompt_bytes, + "request_timeout": args.request_timeout, + "report_interval": args.report_interval, + "invalid_canary_interval": args.invalid_canary_interval, + "max_error_rate": args.max_error_rate, + "server_pid": args.server_pid, + "max_rss_growth_mib": args.max_rss_growth_mib, + "api_key_env": args.api_key_env, + }); + let body = serde_json::to_string_pretty(&config).map_err(|error| error.to_string())?; + fs::write(results_dir.join("config.json"), format!("{body}\n")) + .map_err(|error| error.to_string()) +} + +/// Run the configured soak test and return a process exit code (0 pass, 1 fail). +pub async fn run(args: Args) -> Result { + let token = match &args.api_key_env { + Some(var) => { + let token = std::env::var(var).ok().filter(|value| !value.is_empty()); + if token.is_none() { + return Err(format!("${var} is not set")); + } + token + } + None => None, + }; + + // Per-operation timeouts, not a whole-request deadline: a healthy long stream that keeps + // delivering bytes must not be aborted, so bound connect time and idle time between reads. + let request_timeout = Duration::from_secs_f64(args.request_timeout); + let mut builder = Client::builder() + .no_proxy() + .connect_timeout(request_timeout) + .read_timeout(request_timeout) + .pool_max_idle_per_host(args.concurrency + 4); + if let Some(token) = &token { + let mut headers = HeaderMap::new(); + let value = HeaderValue::from_str(&format!("Bearer {token}")) + .map_err(|_| "the bearer token is not a valid HTTP header value".to_string())?; + headers.insert(AUTHORIZATION, value); + builder = builder.default_headers(headers); + } + let client = builder.build().map_err(|error| error.to_string())?; + let base_url = args.base_url.trim_end_matches('/').to_string(); + + let model = preflight(&client, &base_url, args.model.as_deref()).await?; + let results_dir = args + .results_dir + .clone() + .unwrap_or_else(|| PathBuf::from("soak-results").join(utc_dir_stamp())); + let writer = Arc::new(Mutex::new( + ResultsWriter::new(&results_dir).map_err(|error| error.to_string())?, + )); + write_config(&results_dir, &args, &model)?; + + let endpoints = args.endpoints(); + println!( + "Soak started: model={model} duration={}s concurrency={} endpoints={} results={}", + args.duration, + args.concurrency, + endpoints + .iter() + .map(|endpoint| endpoint.as_str()) + .collect::>() + .join(","), + results_dir.display(), + ); + + let started = Instant::now(); + let stats = Arc::new(Mutex::new(RunStats::new(2026))); + let stop = Arc::new(Stop::new()); + let workers_done = Arc::new(Stop::new()); + let request_numbers = Arc::new(AtomicU64::new(0)); + + spawn_signal_listener(stop.clone()); + + let deadline = { + let stop = stop.clone(); + let stats = stats.clone(); + let duration = Duration::from_secs_f64(args.duration); + tokio::spawn(async move { + tokio::time::sleep(duration).await; + stats.lock().completed_duration = true; + stop.set(); + }) + }; + + let prompt_pool = build_prompt_pool(args.prompt_bytes); + let mut worker_handles = Vec::new(); + for worker_id in 0..args.concurrency { + let handle = tokio::spawn(guard_stop( + stop.clone(), + worker( + client.clone(), + base_url.clone(), + worker_id, + model.clone(), + endpoints.clone(), + prompt_pool.clone(), + args.max_output_tokens, + args.stream_ratio, + stop.clone(), + request_numbers.clone(), + stats.clone(), + writer.clone(), + ), + )); + worker_handles.push((format!("worker-{worker_id}"), handle)); + } + let reporter_handle = tokio::spawn(guard_stop( + stop.clone(), + reporter( + client.clone(), + base_url.clone(), + started, + Duration::from_secs_f64(args.report_interval), + args.duration, + args.server_pid, + stop.clone(), + workers_done.clone(), + stats.clone(), + writer.clone(), + ), + )); + let canary_handle = tokio::spawn(guard_stop( + stop.clone(), + invalid_request_canary( + client.clone(), + base_url.clone(), + args.invalid_canary_interval, + model.clone(), + stop.clone(), + stats.clone(), + writer.clone(), + ), + )); + + stop.wait().await; + deadline.abort(); + + // A crashed worker/reporter/canary must not discard the run: record it as a failure reason + // so the summary is still written and the run fails closed. + let mut task_failures = Vec::new(); + for (name, handle) in worker_handles { + collect_failure(&name, handle.await, &mut task_failures); + } + workers_done.set(); + collect_failure("reporter", reporter_handle.await, &mut task_failures); + collect_failure( + "invalid-request-canary", + canary_handle.await, + &mut task_failures, + ); + + let elapsed = started.elapsed().as_secs_f64(); + let (error_records, dropped_error_records) = { + let writer = writer.lock(); + (writer.error_records, writer.dropped_error_records) + }; + let summary = build_summary( + &stats.lock(), + elapsed, + args.max_error_rate, + args.max_rss_growth_mib, + error_records, + dropped_error_records, + &task_failures, + ); + let latency_block = latency_report(stats.lock().latency_samples()); + let summary_path = results_dir.join("summary.json"); + let summary_body = + serde_json::to_string_pretty(&summary.json).map_err(|error| error.to_string())?; + fs::write(&summary_path, format!("{summary_body}\n")).map_err(|error| error.to_string())?; + + let label = if summary.passed { "PASS" } else { "FAIL" }; + println!( + "Soak {label}: requests={} error_rate={:.4}% p95_ms={} summary={}", + summary.requests, + summary.error_rate * 100.0, + summary + .latency_p95_ms + .map(|value| value.to_string()) + .unwrap_or_default(), + summary_path.display(), + ); + for reason in &summary.reasons { + println!("- {reason}"); + } + if !latency_block.is_empty() { + print!("{latency_block}"); + } + Ok(if summary.passed { 0 } else { 1 }) +} + +/// Parse arguments, run the test on a multi-thread runtime, and map the result to an exit code. +pub fn cli_main() -> ExitCode { + let args = Args::parse(); + if let Err(message) = args.validate() { + eprintln!("switchyard-soak: {message}"); + return ExitCode::from(2); + } + let runtime = match tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + { + Ok(runtime) => runtime, + Err(error) => { + eprintln!("soak test setup failed: {error}"); + return ExitCode::from(2); + } + }; + match runtime.block_on(run(args)) { + Ok(0) => ExitCode::SUCCESS, + Ok(_) => ExitCode::from(1), + Err(error) => { + eprintln!("soak test setup failed: {error}"); + ExitCode::from(2) + } + } +} diff --git a/crates/switchyard-soak/src/main.rs b/crates/switchyard-soak/src/main.rs new file mode 100644 index 000000000..8e3d676d5 --- /dev/null +++ b/crates/switchyard-soak/src/main.rs @@ -0,0 +1,10 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Binary entrypoint for `switchyard-soak`. + +use std::process::ExitCode; + +fn main() -> ExitCode { + switchyard_soak::cli_main() +} diff --git a/crates/switchyard-soak/src/report.rs b/crates/switchyard-soak/src/report.rs new file mode 100644 index 000000000..bfcfe74b7 --- /dev/null +++ b/crates/switchyard-soak/src/report.rs @@ -0,0 +1,355 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Result files plus the background reporter and invalid-request canary tasks. + +use std::fs::{self, File}; +use std::io::{self, Write}; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use parking_lot::Mutex; +use reqwest::Client; +use serde_json::{Value, json}; + +use crate::Stop; +use crate::client::read_server_state; +use crate::stats::{ + RunStats, SERVER_ERRORS_METRIC, SERVER_REQUESTS_METRIC, now_utc_string, percentile, round3, +}; + +/// Cap on individually recorded failures so a bad run cannot fill the disk. +const MAX_ERROR_RECORDS: u64 = 10_000; + +const INTERVAL_FIELDS: [&str; 15] = [ + "timestamp_utc", + "elapsed_seconds", + "requests", + "successes", + "failures", + "requests_per_second", + "latency_p50_ms", + "latency_p95_ms", + "latency_p99_ms", + "latency_max_ms", + "health", + "server_total_requests", + "server_total_errors", + "rss_mib", + "cpu_percent", +]; + +/// Write interval rows and bounded error details as the run proceeds. +pub struct ResultsWriter { + pub results_dir: PathBuf, + interval_file: File, + error_file: File, + pub error_records: u64, + pub dropped_error_records: u64, +} + +impl ResultsWriter { + /// Create a fresh results directory; fails if it already exists. + pub fn new(results_dir: &Path) -> io::Result { + if let Some(parent) = results_dir.parent() + && !parent.as_os_str().is_empty() + { + fs::create_dir_all(parent)?; + } + fs::create_dir(results_dir)?; + let mut interval_file = File::create(results_dir.join("intervals.csv"))?; + writeln!(interval_file, "{}", INTERVAL_FIELDS.join(","))?; + let error_file = File::create(results_dir.join("errors.jsonl"))?; + Ok(Self { + results_dir: results_dir.to_path_buf(), + interval_file, + error_file, + error_records: 0, + dropped_error_records: 0, + }) + } + + /// Append one interval row; `cells` are already formatted in `INTERVAL_FIELDS` order. + pub fn write_interval(&mut self, cells: &[String]) -> io::Result<()> { + writeln!(self.interval_file, "{}", cells.join(","))?; + self.interval_file.flush() + } + + /// Append one failure record, dropping past the bound instead of growing without limit. + pub fn write_error(&mut self, record: &Value) -> io::Result<()> { + if self.error_records >= MAX_ERROR_RECORDS { + self.dropped_error_records += 1; + return Ok(()); + } + // serde_json's map is ordered, so keys serialize sorted, matching the run config file. + writeln!(self.error_file, "{record}")?; + self.error_file.flush()?; + self.error_records += 1; + Ok(()) + } + + /// Overwrite status.json with the current run snapshot, written atomically (write-then-rename) + /// so a remote monitor can read a complete file at any moment without racing the writer. + pub fn write_status(&self, snapshot: &Value) -> io::Result<()> { + let body = serde_json::to_string_pretty(snapshot).map_err(io::Error::other)?; + let tmp = self.results_dir.join("status.json.tmp"); + fs::write(&tmp, format!("{body}\n"))?; + fs::rename(tmp, self.results_dir.join("status.json")) + } +} + +/// Format an optional number for a CSV cell; `None` becomes an empty cell. +fn cell(value: Option) -> String { + value + .map(|value| round3(value).to_string()) + .unwrap_or_default() +} + +/// Return RSS MiB and CPU percent for *pid* using the local `ps` command. +pub async fn process_sample(pid: Option) -> (Option, Option) { + let Some(pid) = pid else { + return (None, None); + }; + let output = match tokio::process::Command::new("ps") + .args(["-o", "rss=,pcpu=", "-p", &pid.to_string()]) + .output() + .await + { + Ok(output) => output, + // ps missing or fork failed: record a process-check miss, don't crash the reporter. + Err(_) => return (None, None), + }; + let fields: Vec = String::from_utf8_lossy(&output.stdout) + .split_whitespace() + .map(str::to_string) + .collect(); + if !output.status.success() || fields.len() != 2 { + return (None, None); + } + match (fields[0].parse::(), fields[1].parse::()) { + (Ok(rss_kib), Ok(cpu_percent)) => (Some(rss_kib / 1024.0), Some(cpu_percent)), + _ => (None, None), + } +} + +/// Write one liveness, metrics, resource, and latency row per interval. +#[allow(clippy::too_many_arguments)] +pub async fn reporter( + client: Client, + base_url: String, + started: Instant, + interval: Duration, + target_seconds: f64, + server_pid: Option, + stop: Arc, + workers_done: Arc, + stats: Arc>, + writer: Arc>, +) -> Result<(), String> { + let mut previous_report = started; + loop { + tokio::select! { + _ = tokio::time::sleep(interval) => {} + _ = stop.wait() => {} + } + // Once stopping, wait for the workers to drain so the final row counts their last requests. + if stop.is_set() { + workers_done.wait().await; + } + + let now = Instant::now(); + let interval_stats = stats.lock().take_interval(); + let (healthy, metrics) = read_server_state(&client, &base_url).await; + let (rss_mib, cpu_percent) = process_sample(server_pid).await; + let server_requests = metrics.get(SERVER_REQUESTS_METRIC).copied(); + let server_errors = metrics.get(SERVER_ERRORS_METRIC).copied(); + + let (total_successes, total_failures, server_restarts, completed_duration) = { + let mut state = stats.lock(); + state.health_checks += 1; + state.metrics_checks += 1; + if !healthy { + state.health_failures += 1; + } + if server_requests.is_none() || server_errors.is_none() { + state.metrics_failures += 1; + } + if server_pid.is_some() { + state.process_checks += 1; + if rss_mib.is_none() || cpu_percent.is_none() { + state.process_failures += 1; + } + } + if let Some(rss) = rss_mib { + state.rss_samples.push(rss); + } + if let (Some(current), Some(previous)) = + (server_requests, state.previous_server_requests) + && current < previous + { + state.server_restarts += 1; + } + if let Some(current) = server_requests { + state.previous_server_requests = Some(current); + } + ( + state.total_successes, + state.total_failures, + state.server_restarts, + state.completed_duration, + ) + }; + + let elapsed_interval = (now - previous_report).as_secs_f64().max(0.001); + let requests = interval_stats.successes + interval_stats.failures; + let latency_max = interval_stats + .latencies_ms + .iter() + .copied() + .fold(None, |acc: Option, value| { + Some(acc.map_or(value, |m: f64| m.max(value))) + }); + let requests_per_second = round3(requests as f64 / elapsed_interval); + let timestamp = now_utc_string(); + let elapsed_seconds = round3((now - started).as_secs_f64()); + let health_label = if healthy { "ok" } else { "failed" }; + let latency_p95 = percentile(&interval_stats.latencies_ms, 0.95).map(round3); + + let cells = vec![ + timestamp.clone(), + elapsed_seconds.to_string(), + requests.to_string(), + interval_stats.successes.to_string(), + interval_stats.failures.to_string(), + requests_per_second.to_string(), + cell(percentile(&interval_stats.latencies_ms, 0.50)), + cell(latency_p95), + cell(percentile(&interval_stats.latencies_ms, 0.99)), + cell(latency_max), + health_label.to_string(), + cell(server_requests), + cell(server_errors), + cell(rss_mib), + cell(cpu_percent), + ]; + // Cumulative, progress, and a glanceable status token so a remote tail of the log shows + // at once that the run is alive, how far along it is, and whether it is healthy. + let cumulative_requests = total_successes + total_failures; + let cumulative_error_rate = if cumulative_requests > 0 { + total_failures as f64 / cumulative_requests as f64 + } else { + 0.0 + }; + let progress = if target_seconds > 0.0 { + (elapsed_seconds / target_seconds).min(1.0) + } else { + 0.0 + }; + let status = if requests == 0 { + "stalled" + } else if !healthy || interval_stats.failures > 0 { + "degraded" + } else { + "ok" + }; + let p95_text = latency_p95.map(|v| v.to_string()).unwrap_or_default(); + let rss_text = rss_mib.map(|v| round3(v).to_string()).unwrap_or_default(); + + let snapshot = json!({ + "timestamp_utc": timestamp, + "elapsed_seconds": elapsed_seconds, + "target_seconds": round3(target_seconds), + "progress": round3(progress), + "requests": cumulative_requests, + "successes": total_successes, + "failures": total_failures, + "error_rate": cumulative_error_rate, + "interval_requests": requests, + "requests_per_second": requests_per_second, + "latency_p95_ms": latency_p95, + "health": health_label, + "rss_mib": rss_mib.map(round3), + "detected_server_restarts": server_restarts, + "status": status, + "completed_duration": completed_duration, + }); + { + let mut w = writer.lock(); + w.write_interval(&cells) + .map_err(|error| error.to_string())?; + w.write_status(&snapshot) + .map_err(|error| error.to_string())?; + } + + println!( + "[{timestamp}] progress={elapsed_seconds:.0}s/{target_seconds:.0}s({:.0}%) \ + reqs={cumulative_requests} interval={requests} errors={total_failures}({:.4}%) \ + rps={requests_per_second} p95_ms={p95_text} health={health_label} rss_mib={rss_text} \ + status={}", + progress * 100.0, + cumulative_error_rate * 100.0, + status.to_uppercase(), + ); + + previous_report = now; + if stop.is_set() { + return Ok(()); + } + } +} + +/// Confirm invalid input returns 400 and the server stays live, on a fixed interval. +pub async fn invalid_request_canary( + client: Client, + base_url: String, + interval: f64, + model: String, + stop: Arc, + stats: Arc>, + writer: Arc>, +) -> Result<(), String> { + if interval <= 0.0 { + return Ok(()); + } + let interval = Duration::from_secs_f64(interval); + loop { + tokio::select! { + _ = tokio::time::sleep(interval) => {} + _ = stop.wait() => {} + } + if stop.is_set() { + return Ok(()); + } + + stats.lock().canaries += 1; + let probe = async { + let invalid = client + .post(format!("{base_url}/v1/chat/completions")) + .json(&json!({"model": model, "messages": []})) + .send() + .await?; + let health = client.get(format!("{base_url}/health")).send().await?; + Ok::<(u16, u16), reqwest::Error>((invalid.status().as_u16(), health.status().as_u16())) + } + .await; + let (passed, detail) = match probe { + Ok((invalid_status, health_status)) => ( + invalid_status == 400 && health_status == 200, + format!("invalid_status={invalid_status}, health_status={health_status}"), + ), + Err(error) => (false, error.to_string()), + }; + if !passed { + stats.lock().canary_failures += 1; + writer + .lock() + .write_error(&json!({ + "timestamp_utc": now_utc_string(), + "error": "invalid_request_canary", + "detail": detail, + })) + .map_err(|error| error.to_string())?; + } + } +} diff --git a/crates/switchyard-soak/src/stats.rs b/crates/switchyard-soak/src/stats.rs new file mode 100644 index 000000000..57ed29f73 --- /dev/null +++ b/crates/switchyard-soak/src/stats.rs @@ -0,0 +1,560 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Bounded in-memory run state, latency percentiles, and the final pass/fail summary. + +use std::collections::BTreeMap; +use std::time::{SystemTime, UNIX_EPOCH}; + +use rand::rngs::StdRng; +use rand::{RngExt, SeedableRng}; +use serde_json::{Value, json}; + +/// Cap on retained latency samples; a long run stays within this bound by reservoir sampling. +const RESERVOIR_SIZE: usize = 100_000; + +/// Cumulative counter names Switchyard exposes on `/metrics`. +pub const SERVER_REQUESTS_METRIC: &str = "switchyard_total_requests"; +pub const SERVER_ERRORS_METRIC: &str = "switchyard_total_errors"; + +/// Round to three decimals so result files stay readable. +pub fn round3(value: f64) -> f64 { + (value * 1000.0).round() / 1000.0 +} + +/// Parse a duration such as `30s`, `15m`, or `48h` into seconds. +pub fn parse_duration(value: &str) -> Result { + let text = value.trim().to_lowercase(); + let bad = || "duration must use s, m, or h, for example 30s or 48h".to_string(); + let unit = text.chars().last().ok_or_else(bad)?; + let multiplier = match unit { + 's' => 1.0, + 'm' => 60.0, + 'h' => 3600.0, + _ => return Err(bad()), + }; + // Accept only `\d+(\.\d+)?` before the unit: at least one digit, at most one dot with + // digits on both sides. This rejects "10", "5x", "-3s", "1.", and ".5". + let number = &text[..text.len() - unit.len_utf8()]; + let parts: Vec<&str> = number.split('.').collect(); + let well_formed = matches!(parts.len(), 1 | 2) + && parts + .iter() + .all(|part| !part.is_empty() && part.bytes().all(|b| b.is_ascii_digit())); + if !well_formed { + return Err(bad()); + } + let seconds = number.parse::().map_err(|_| bad())? * multiplier; + if seconds <= 0.0 { + return Err("duration must be greater than zero".to_string()); + } + Ok(seconds) +} + +/// Return the nearest-rank percentile for *values*, or `None` when empty. +pub fn percentile(values: &[f64], quantile: f64) -> Option { + if values.is_empty() { + return None; + } + let mut ordered = values.to_vec(); + ordered.sort_by(|a, b| a.total_cmp(b)); + // round_ties_even matches Python's round(), which breaks exact halves to even. + let index = (((ordered.len() - 1) as f64) * quantile).round_ties_even() as usize; + Some(ordered[index]) +} + +/// One equal-width latency bucket for the end-of-run histogram. +struct Bucket { + low: f64, + high: f64, + count: usize, +} + +/// Split *values* into *bucket_count* equal-width buckets between the min and max sample. +/// The maximum sample falls in the last bucket rather than spilling past it. +fn histogram(values: &[f64], bucket_count: usize) -> Vec { + let min = values.iter().copied().fold(f64::INFINITY, f64::min); + let max = values.iter().copied().fold(f64::NEG_INFINITY, f64::max); + let width = ((max - min) / bucket_count as f64).max(f64::MIN_POSITIVE); + let mut counts = vec![0usize; bucket_count]; + for &value in values { + let index = (((value - min) / width) as usize).min(bucket_count - 1); + counts[index] += 1; + } + counts + .into_iter() + .enumerate() + .map(|(index, count)| Bucket { + low: min + index as f64 * width, + high: min + (index + 1) as f64 * width, + count, + }) + .collect() +} + +/// Render an oha-style latency percentile table and ASCII histogram, or empty when no samples. +/// Built from the bounded reservoir, so it reflects the whole run within the reservoir cap. +pub fn latency_report(reservoir: &[f64]) -> String { + if reservoir.is_empty() { + return String::new(); + } + let mut out = String::from("Latency distribution (ms):\n"); + for (label, quantile) in [ + ("p50", 0.50), + ("p75", 0.75), + ("p90", 0.90), + ("p95", 0.95), + ("p99", 0.99), + ("p99.9", 0.999), + ] { + if let Some(value) = percentile(reservoir, quantile) { + out.push_str(&format!(" {label:<6}{:>12.3}\n", round3(value))); + } + } + out.push_str("Latency histogram (ms):\n"); + let buckets = histogram(reservoir, 10); + let widest = buckets + .iter() + .map(|bucket| bucket.count) + .max() + .unwrap_or(1) + .max(1); + for bucket in &buckets { + let bar = "\u{25a0}".repeat(bucket.count * 40 / widest); + out.push_str(&format!( + " {:>9.1} - {:>9.1} [{:>7}] {bar}\n", + round3(bucket.low), + round3(bucket.high), + bucket.count + )); + } + out +} + +fn rounded_percentile(values: &[f64], quantile: f64) -> Option { + percentile(values, quantile).map(round3) +} + +/// Return four stable prompts with reusable prefixes of *prompt_bytes* filler each. +pub fn build_prompt_pool(prompt_bytes: usize) -> Vec { + (0..4) + .map(|index| { + let prefix = format!("Switchyard soak prefix {index}. "); + let instruction = "Reply with exactly OK. "; + let unit = "load test context "; + let mut filler = unit.repeat(prompt_bytes / unit.len() + 1); + filler.truncate(prompt_bytes); + format!("{prefix}{filler}{instruction}") + }) + .collect() +} + +/// ISO-8601 UTC timestamp at second precision, e.g. `2026-07-30T18:04:00Z`. +pub fn now_utc_string() -> String { + format_utc(SystemTime::now()) +} + +/// Compact UTC stamp for a results directory, e.g. `20260730T180400Z`. +pub fn utc_dir_stamp() -> String { + let secs = unix_seconds(SystemTime::now()); + let (year, month, day, hour, minute, second) = civil_from_unix(secs); + format!("{year:04}{month:02}{day:02}T{hour:02}{minute:02}{second:02}Z") +} + +fn format_utc(time: SystemTime) -> String { + let (year, month, day, hour, minute, second) = civil_from_unix(unix_seconds(time)); + format!("{year:04}-{month:02}-{day:02}T{hour:02}:{minute:02}:{second:02}Z") +} + +fn unix_seconds(time: SystemTime) -> i64 { + // Before the epoch never happens for a live run; clamp to 0 rather than panic. + time.duration_since(UNIX_EPOCH) + .map(|d| d.as_secs() as i64) + .unwrap_or(0) +} + +/// Split seconds-since-epoch into UTC (year, month, day, hour, minute, second). +fn civil_from_unix(secs: i64) -> (i64, u32, u32, u32, u32, u32) { + let days = secs.div_euclid(86_400); + let rem = secs.rem_euclid(86_400); + let (hour, minute, second) = (rem / 3600, rem % 3600 / 60, rem % 60); + // Howard Hinnant's days-to-civil algorithm. + let z = days + 719_468; + let era = if z >= 0 { z } else { z - 146_096 } / 146_097; + let doe = z - era * 146_097; + let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365; + let year = yoe + era * 400; + let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + let mp = (5 * doy + 2) / 153; + let day = (doy - (153 * mp + 2) / 5 + 1) as u32; + let month = if mp < 10 { mp + 3 } else { mp - 9 } as u32; + ( + year + i64::from(month <= 2), + month, + day, + hour as u32, + minute as u32, + second as u32, + ) +} + +/// Request results collected since the previous report. +#[derive(Default)] +pub struct IntervalStats { + pub successes: u64, + pub failures: u64, + pub latencies_ms: Vec, +} + +/// Bounded in-memory state for one soak run. +pub struct RunStats { + rng: StdRng, + interval: IntervalStats, + pub total_successes: u64, + pub total_failures: u64, + pub endpoint_successes: BTreeMap, + pub endpoint_failures: BTreeMap, + pub error_kinds: BTreeMap, + latency_reservoir: Vec, + latency_count: u64, + pub health_checks: u64, + pub health_failures: u64, + pub metrics_checks: u64, + pub metrics_failures: u64, + pub process_checks: u64, + pub process_failures: u64, + pub canaries: u64, + pub canary_failures: u64, + pub server_restarts: u64, + pub previous_server_requests: Option, + pub rss_samples: Vec, + pub completed_duration: bool, +} + +impl RunStats { + pub fn new(seed: u64) -> Self { + Self { + rng: StdRng::seed_from_u64(seed), + interval: IntervalStats::default(), + total_successes: 0, + total_failures: 0, + endpoint_successes: BTreeMap::new(), + endpoint_failures: BTreeMap::new(), + error_kinds: BTreeMap::new(), + latency_reservoir: Vec::new(), + latency_count: 0, + health_checks: 0, + health_failures: 0, + metrics_checks: 0, + metrics_failures: 0, + process_checks: 0, + process_failures: 0, + canaries: 0, + canary_failures: 0, + server_restarts: 0, + previous_server_requests: None, + rss_samples: Vec::new(), + completed_duration: false, + } + } + + /// Record one completed inference request; `error_kind` is `None` on success. + pub fn record(&mut self, endpoint: &str, latency_ms: f64, error_kind: Option<&str>) { + match error_kind { + None => { + self.interval.successes += 1; + self.total_successes += 1; + *self + .endpoint_successes + .entry(endpoint.to_string()) + .or_default() += 1; + } + Some(kind) => { + self.interval.failures += 1; + self.total_failures += 1; + *self + .endpoint_failures + .entry(endpoint.to_string()) + .or_default() += 1; + *self.error_kinds.entry(kind.to_string()).or_default() += 1; + } + } + self.interval.latencies_ms.push(latency_ms); + self.latency_count += 1; + if self.latency_reservoir.len() < RESERVOIR_SIZE { + self.latency_reservoir.push(latency_ms); + } else { + let index = self.rng.random_range(0..self.latency_count) as usize; + if index < RESERVOIR_SIZE { + self.latency_reservoir[index] = latency_ms; + } + } + } + + /// Return and reset the current interval. + pub fn take_interval(&mut self) -> IntervalStats { + std::mem::take(&mut self.interval) + } + + /// The bounded reservoir of latency samples, for the end-of-run distribution report. + pub fn latency_samples(&self) -> &[f64] { + &self.latency_reservoir + } +} + +/// Final result of a run: the exit-relevant fields plus the JSON written to `summary.json`. +pub struct Summary { + pub passed: bool, + pub reasons: Vec, + pub requests: u64, + pub error_rate: f64, + pub latency_p95_ms: Option, + pub json: Value, +} + +/// Build the final result and its release-gate reasons. +pub fn build_summary( + stats: &RunStats, + elapsed_seconds: f64, + max_error_rate: f64, + max_rss_growth_mib: Option, + error_records: u64, + dropped_error_records: u64, + task_failures: &[String], +) -> Summary { + let total = stats.total_successes + stats.total_failures; + let error_rate = if total > 0 { + stats.total_failures as f64 / total as f64 + } else { + 1.0 + }; + let rss_first = stats.rss_samples.first().copied(); + let rss_last = stats.rss_samples.last().copied(); + let rss_growth = match (rss_first, rss_last) { + (Some(first), Some(last)) => Some(last - first), + _ => None, + }; + + let mut reasons: Vec = task_failures.to_vec(); + if !stats.completed_duration { + reasons.push("the run stopped before the requested duration".to_string()); + } + if total == 0 { + reasons.push("no inference requests completed".to_string()); + } + if error_rate > max_error_rate { + reasons.push(format!( + "request error rate {:.4}% exceeded the {:.4}% limit", + error_rate * 100.0, + max_error_rate * 100.0 + )); + } + if stats.health_failures > 0 { + reasons.push(format!("{} liveness checks failed", stats.health_failures)); + } + if stats.metrics_failures > 0 { + reasons.push(format!( + "{} server metrics checks failed", + stats.metrics_failures + )); + } + if stats.process_failures > 0 { + reasons.push(format!( + "{} server process checks failed", + stats.process_failures + )); + } + if stats.canary_failures > 0 { + reasons.push(format!( + "{} invalid-request recovery checks failed", + stats.canary_failures + )); + } + if stats.server_restarts > 0 { + reasons.push(format!( + "server counters reset {} time(s)", + stats.server_restarts + )); + } + if let (Some(limit), Some(growth)) = (max_rss_growth_mib, rss_growth) + && growth > limit + { + reasons.push(format!( + "server RSS grew {growth:.1} MiB, above the {limit:.1} MiB limit" + )); + } + + let rss_max = stats + .rss_samples + .iter() + .copied() + .fold(None, |acc: Option, value| { + Some(acc.map_or(value, |m| m.max(value))) + }); + let requests_per_second = if elapsed_seconds > 0.0 { + total as f64 / elapsed_seconds + } else { + 0.0 + }; + + let json = json!({ + "passed": reasons.is_empty(), + "failure_reasons": reasons, + "completed_duration": stats.completed_duration, + "elapsed_seconds": round3(elapsed_seconds), + "requests": total, + "successes": stats.total_successes, + "failures": stats.total_failures, + "error_rate": error_rate, + "requests_per_second": requests_per_second, + "endpoint_successes": stats.endpoint_successes, + "endpoint_failures": stats.endpoint_failures, + "error_kinds": stats.error_kinds, + "latency_p50_ms": rounded_percentile(&stats.latency_reservoir, 0.50), + "latency_p95_ms": rounded_percentile(&stats.latency_reservoir, 0.95), + "latency_p99_ms": rounded_percentile(&stats.latency_reservoir, 0.99), + "health_checks": stats.health_checks, + "health_failures": stats.health_failures, + "metrics_checks": stats.metrics_checks, + "metrics_failures": stats.metrics_failures, + "process_checks": stats.process_checks, + "process_failures": stats.process_failures, + "invalid_request_canaries": stats.canaries, + "invalid_request_canary_failures": stats.canary_failures, + "detected_server_restarts": stats.server_restarts, + "rss_first_mib": rss_first, + "rss_last_mib": rss_last, + "rss_max_mib": rss_max, + "rss_growth_mib": rss_growth, + "error_records": error_records, + "dropped_error_records": dropped_error_records, + }); + + Summary { + passed: reasons.is_empty(), + reasons, + requests: total, + error_rate, + latency_p95_ms: rounded_percentile(&stats.latency_reservoir, 0.95), + json, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_duration_reads_suffixes() { + assert_eq!(parse_duration("30s"), Ok(30.0)); + assert_eq!(parse_duration("2.5m"), Ok(150.0)); + assert_eq!(parse_duration("48h"), Ok(172_800.0)); + } + + #[test] + fn parse_duration_rejects_bad_values() { + for value in ["0s", "10", "5x", "-3s", "1d", "1.", ".5"] { + assert!(parse_duration(value).is_err(), "{value} should be rejected"); + } + } + + #[test] + fn percentile_uses_nearest_rank() { + let values = [1.0, 2.0, 3.0, 4.0]; + assert_eq!(percentile(&values, 0.0), Some(1.0)); + assert_eq!(percentile(&values, 0.95), Some(4.0)); + assert_eq!(percentile(&[], 0.5), None); + // Exact half -> even rank, matching Python round(): (6-1)*0.5 = 2.5 rounds to 2, not 3. + assert_eq!( + percentile(&[10.0, 20.0, 30.0, 40.0, 50.0, 60.0], 0.5), + Some(30.0) + ); + } + + #[test] + fn format_utc_matches_known_instants() { + let epoch = SystemTime::UNIX_EPOCH; + assert_eq!(format_utc(epoch), "1970-01-01T00:00:00Z"); + // 2023-11-14T22:13:20Z + let later = epoch + std::time::Duration::from_secs(1_700_000_000); + assert_eq!(format_utc(later), "2023-11-14T22:13:20Z"); + } + + #[test] + fn histogram_buckets_cover_all_samples() { + let values = [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0]; + let buckets = histogram(&values, 10); + assert_eq!(buckets.len(), 10); + // Every sample is counted exactly once, and the max lands in the last bucket. + assert_eq!(buckets.iter().map(|b| b.count).sum::(), values.len()); + assert_eq!(buckets[9].count, 2); // 9.0 and 10.0 (the max) share the last bucket + assert_eq!(buckets[0].low, 0.0); + + // A single repeated value does not panic and lands entirely in one bucket. + let flat = histogram(&[5.0, 5.0, 5.0], 10); + assert_eq!(flat.iter().map(|b| b.count).sum::(), 3); + } + + #[test] + fn latency_report_is_empty_without_samples() { + assert_eq!(latency_report(&[]), ""); + assert!(latency_report(&[1.0, 2.0, 3.0]).contains("Latency histogram (ms):")); + } + + #[test] + fn build_prompt_pool_sizes_filler() { + let pool = build_prompt_pool(32); + assert_eq!(pool.len(), 4); + assert!(pool[0].starts_with("Switchyard soak prefix 0. ")); + assert!(pool[0].ends_with("Reply with exactly OK. ")); + } + + #[test] + fn summary_fails_on_restart_and_error_budget() { + let mut stats = RunStats::new(1); + stats.completed_duration = true; + stats.total_successes = 998; + stats.total_failures = 2; + stats.server_restarts = 1; + + let summary = build_summary(&stats, 10.0, 0.001, None, 0, 0, &[]); + + assert!(!summary.passed); + assert!(summary.reasons.iter().any(|r| r.contains("error rate"))); + assert!(summary.reasons.iter().any(|r| r.contains("counters reset"))); + } + + #[test] + fn summary_records_task_failures() { + let mut stats = RunStats::new(1); + stats.completed_duration = true; + stats.total_successes = 100; + + let summary = build_summary( + &stats, + 10.0, + 0.0, + None, + 0, + 0, + &["reporter failed: boom".to_string()], + ); + + assert!(!summary.passed); + assert!(summary.reasons.iter().any(|r| r == "reporter failed: boom")); + } + + #[test] + fn summary_flags_rss_growth() { + let mut stats = RunStats::new(1); + stats.completed_duration = true; + stats.total_successes = 100; + stats.rss_samples = vec![100.0, 700.0]; + + let summary = build_summary(&stats, 10.0, 0.0, Some(512.0), 0, 0, &[]); + + assert!(!summary.passed); + assert_eq!(summary.json["rss_growth_mib"], json!(600.0)); + assert!(summary.reasons.iter().any(|r| r.contains("RSS grew"))); + } +} diff --git a/crates/switchyard-soak/tests/soak.rs b/crates/switchyard-soak/tests/soak.rs new file mode 100644 index 000000000..6c28dda9c --- /dev/null +++ b/crates/switchyard-soak/tests/soak.rs @@ -0,0 +1,318 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! End-to-end tests that drive the soak runner against a hermetic axum mock of a Switchyard server. + +use std::error::Error; +use std::sync::Arc; +use std::time::Duration; + +use axum::Router; +use axum::extract::Json; +use axum::http::{StatusCode, header}; +use axum::response::{IntoResponse, Response}; +use axum::routing::{get, post}; +use clap::Parser; +use parking_lot::Mutex; +use serde_json::{Value, json}; + +use switchyard_soak::client::{Endpoint, preflight, read_server_state, request_body, send_request}; +use switchyard_soak::report::{ResultsWriter, invalid_request_canary}; +use switchyard_soak::stats::RunStats; +use switchyard_soak::{Args, Stop, run}; + +type TestResult = Result<(), Box>; + +/// Bind an ephemeral port, serve *app*, and return its base URL. The listener is bound before the +/// server task starts, so client connections queue rather than being refused. +async fn serve(app: Router) -> Result> { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let addr = listener.local_addr()?; + tokio::spawn(async move { + let _ = axum::serve(listener, app.into_make_service()).await; + }); + Ok(format!("http://{addr}")) +} + +fn client() -> Result> { + Ok(reqwest::Client::builder().no_proxy().build()?) +} + +fn sse_body(body: &'static str) -> Response { + ([(header::CONTENT_TYPE, "text/event-stream")], body).into_response() +} + +fn is_stream(body: &Value) -> bool { + body.get("stream").and_then(Value::as_bool).unwrap_or(false) +} + +/// A well-behaved Switchyard: valid shapes, SSE for streaming, HTTP 400 for empty messages. +fn healthy_switchyard() -> Router { + async fn health() -> Response { + Json(json!({"status": "ok"})).into_response() + } + async fn metrics() -> Response { + "switchyard_total_requests 1\nswitchyard_total_errors 0\n".into_response() + } + async fn models() -> Response { + Json(json!({"data": [{"id": "soak-route"}]})).into_response() + } + async fn chat(Json(body): Json) -> Response { + let empty = body + .get("messages") + .and_then(Value::as_array) + .map(|messages| messages.is_empty()) + .unwrap_or(false); + if empty { + return (StatusCode::BAD_REQUEST, "messages must not be empty").into_response(); + } + if is_stream(&body) { + sse_body("data: {}\n\ndata: [DONE]\n\n") + } else { + Json(json!({"choices": []})).into_response() + } + } + async fn messages(Json(body): Json) -> Response { + if is_stream(&body) { + sse_body("data: {}\n\ndata: [DONE]\n\n") + } else { + Json(json!({"content": []})).into_response() + } + } + async fn responses(Json(body): Json) -> Response { + if is_stream(&body) { + sse_body("data: {}\n\ndata: [DONE]\n\n") + } else { + Json(json!({"output": []})).into_response() + } + } + Router::new() + .route("/health", get(health)) + .route("/metrics", get(metrics)) + .route("/v1/models", get(models)) + .route("/v1/chat/completions", post(chat)) + .route("/v1/messages", post(messages)) + .route("/v1/responses", post(responses)) +} + +#[tokio::test] +async fn short_run_passes_against_live_switchyard_routes() -> TestResult { + let base_url = serve(healthy_switchyard()).await?; + let dir = tempfile::tempdir()?; + let results_dir = dir.path().join("soak-results"); + + let args = Args::try_parse_from([ + "switchyard-soak", + "--base-url", + &base_url, + "--model", + "soak-route", + "--duration", + "1s", + "--concurrency", + "2", + "--stream-ratio", + "0.5", + "--report-interval", + "0.2", + "--invalid-canary-interval", + "0.2", + "--results-dir", + results_dir.to_str().ok_or("non-utf8 results dir")?, + ])?; + args.validate()?; + assert_eq!(run(args).await?, 0); + + let summary: Value = + serde_json::from_str(&std::fs::read_to_string(results_dir.join("summary.json"))?)?; + assert_eq!(summary["passed"], json!(true)); + for endpoint in ["chat", "messages", "responses"] { + assert!( + summary["endpoint_successes"][endpoint] + .as_u64() + .unwrap_or(0) + > 0, + "expected successes on {endpoint}: {summary}" + ); + } + assert!(summary["invalid_request_canaries"].as_u64().unwrap_or(0) > 0); + Ok(()) +} + +#[tokio::test] +async fn preflight_selects_default_model() -> TestResult { + async fn models() -> Response { + Json(json!({"data": [{"id": "route-a"}, {"id": "route-b"}], "default_model": "route-b"})) + .into_response() + } + let app = Router::new() + .route("/health", get(|| async { Json(json!({"status": "ok"})) })) + .route("/v1/models", get(models)); + let base_url = serve(app).await?; + + assert_eq!(preflight(&client()?, &base_url, None).await?, "route-b"); + Ok(()) +} + +#[tokio::test] +async fn preflight_rejects_empty_and_unknown_model() -> TestResult { + let empty = Router::new() + .route("/health", get(|| async { Json(json!({"status": "ok"})) })) + .route("/v1/models", get(|| async { Json(json!({"data": []})) })); + let base_url = serve(empty).await?; + let error = preflight(&client()?, &base_url, None).await.unwrap_err(); + assert!(error.contains("no model to test"), "{error}"); + + let one = Router::new() + .route("/health", get(|| async { Json(json!({"status": "ok"})) })) + .route( + "/v1/models", + get(|| async { Json(json!({"data": [{"id": "route-a"}]})) }), + ); + let base_url = serve(one).await?; + let error = preflight(&client()?, &base_url, Some("missing-model")) + .await + .unwrap_err(); + assert!(error.contains("is not listed"), "{error}"); + Ok(()) +} + +#[tokio::test] +async fn send_request_accepts_json_and_streaming_success() -> TestResult { + let base_url = serve(healthy_switchyard()).await?; + let http = client()?; + + let (error, _) = send_request( + &http, + &base_url, + Endpoint::Chat, + &request_body(Endpoint::Chat, "route", "hi", 8, false), + ) + .await; + assert_eq!(error, None); + let (error, _) = send_request( + &http, + &base_url, + Endpoint::Chat, + &request_body(Endpoint::Chat, "route", "hi", 8, true), + ) + .await; + assert_eq!(error, None); + Ok(()) +} + +#[tokio::test] +async fn send_request_rejects_missing_fields_and_non_sse_stream() -> TestResult { + async fn chat(Json(body): Json) -> Response { + if is_stream(&body) { + // 200 but application/json instead of an event stream. + Json(json!({"choices": []})).into_response() + } else { + // 200 without the required "choices" field. + Json(json!({})).into_response() + } + } + let base_url = serve(Router::new().route("/v1/chat/completions", post(chat))).await?; + let http = client()?; + + let (error, _) = send_request( + &http, + &base_url, + Endpoint::Chat, + &request_body(Endpoint::Chat, "route", "hi", 8, false), + ) + .await; + assert_eq!(error.as_deref(), Some("invalid_response")); + let (error, _) = send_request( + &http, + &base_url, + Endpoint::Chat, + &request_body(Endpoint::Chat, "route", "hi", 8, true), + ) + .await; + assert_eq!(error.as_deref(), Some("invalid_stream")); + Ok(()) +} + +#[tokio::test] +async fn send_request_reports_http_error_and_empty_stream() -> TestResult { + async fn chat(Json(body): Json) -> Response { + if is_stream(&body) { + sse_body("\n\n") // event stream with no data + } else { + (StatusCode::BAD_REQUEST, "bad request").into_response() + } + } + let base_url = serve(Router::new().route("/v1/chat/completions", post(chat))).await?; + let http = client()?; + + let (error, _) = send_request( + &http, + &base_url, + Endpoint::Chat, + &request_body(Endpoint::Chat, "route", "hi", 8, false), + ) + .await; + assert_eq!(error.as_deref(), Some("http_400")); + let (error, _) = send_request( + &http, + &base_url, + Endpoint::Chat, + &request_body(Endpoint::Chat, "route", "hi", 8, true), + ) + .await; + assert_eq!(error.as_deref(), Some("empty_stream")); + Ok(()) +} + +#[tokio::test] +async fn read_server_state_treats_non_dict_health_as_unhealthy() -> TestResult { + let app = Router::new() + // Valid JSON, but a list rather than an object. + .route("/health", get(|| async { Json(json!(["ok"])) })) + .route( + "/metrics", + get(|| async { "switchyard_total_requests 5\nswitchyard_total_errors 0\n" }), + ); + let base_url = serve(app).await?; + + let (healthy, metrics) = read_server_state(&client()?, &base_url).await; + assert!(!healthy); + assert_eq!(metrics.get("switchyard_total_requests"), Some(&5.0)); + Ok(()) +} + +#[tokio::test] +async fn invalid_request_canary_flags_non_400() -> TestResult { + let app = Router::new() + .route("/health", get(|| async { Json(json!({"status": "ok"})) })) + // Should have been HTTP 400 for an invalid request. + .route( + "/v1/chat/completions", + post(|| async { Json(json!({"choices": []})) }), + ); + let base_url = serve(app).await?; + + let dir = tempfile::tempdir()?; + let stats = Arc::new(Mutex::new(RunStats::new(0))); + let writer = Arc::new(Mutex::new(ResultsWriter::new(&dir.path().join("results"))?)); + let stop = Arc::new(Stop::new()); + + let task = tokio::spawn(invalid_request_canary( + client()?, + base_url, + 0.05, + "soak-route".to_string(), + stop.clone(), + stats.clone(), + writer.clone(), + )); + tokio::time::sleep(Duration::from_millis(120)).await; + stop.set(); + task.await??; + + let stats = stats.lock(); + assert!(stats.canaries >= 1); + assert_eq!(stats.canary_failures, stats.canaries); + Ok(()) +} diff --git a/docs/operations/soak_test.md b/docs/operations/soak_test.md new file mode 100644 index 000000000..111e363ac --- /dev/null +++ b/docs/operations/soak_test.md @@ -0,0 +1,194 @@ +# Soak test a release candidate + +A Switchyard soak test sends sustained traffic through a release-candidate +server long enough to expose failures that short tests miss. Run it for 48 +hours before code freeze when a release changes libsy, the Rust server, +routing, streaming, translation, or server lifecycle behavior. + +The test sends closed-loop traffic through: + +- OpenAI Chat Completions (`/v1/chat/completions`) +- Anthropic Messages (`/v1/messages`) +- OpenAI Responses (`/v1/responses`) +- streaming and non-streaming responses +- four repeated prompt prefixes + +The runner also checks `/health` and `/metrics` every minute. Every five +minutes, it sends an invalid Chat Completions request, expects HTTP 400, and +then confirms the server is still live. + +## Prepare the server + +Run the exact commit, build, route bundle, backend, and model planned for the +release. Do not use a development server in front of a different Switchyard +build. + +For the Python server: + +```bash +uv sync +uv run switchyard --routing-profiles release-routes.yaml -- serve --port 4000 \ + > switchyard-soak.log 2>&1 & +SOAK_SERVER_PID=$! +``` + +For the Rust server and its libsy algorithms: + +```bash +cargo build --release -p switchyard-server +target/release/switchyard-server --config release-routes.toml \ + > switchyard-soak.log 2>&1 & +SOAK_SERVER_PID=$! +``` + +Wait for `GET http://127.0.0.1:4000/health` to return HTTP 200 with the JSON +body `{"status": "ok"}`; the runner requires both before it starts. Check +`GET http://127.0.0.1:4000/v1/models` and choose the model id that represents +the release workload. + +Run the server and test from a dedicated host, job scheduler, or terminal +multiplexer that will stay alive for the full test. Confirm that the host will +not suspend or restart and has enough disk space for the server log. + +## Build the soak tester + +The soak tester is a Rust binary. Build it once from the same checkout as the +release, then run it directly: + +```bash +cargo build --release -p switchyard-soak +``` + +## Rehearse against a mock backend + +Rehearse the soak with no live-provider credentials or cost by running +[VidaiMock](https://github.com/vidaiUK/VidaiMock) — an Apache-2.0 mock LLM server that speaks the +OpenAI, Anthropic, and Responses wire formats with realistic streaming — as the backend behind +`switchyard-server`. `scripts/soak-rehearsal.sh` starts the mock, points a passthrough +`switchyard-server` at it, waits for both to report healthy, and runs the soak against the local +stack: + +```bash +# Requires the vidaimock binary on PATH (github.com/vidaiUK/VidaiMock/releases) and release builds +# of switchyard-server and switchyard-soak. +scripts/soak-rehearsal.sh --duration 5m --concurrency 8 --invalid-canary-interval 0 +``` + +Disable the invalid-request canary (`--invalid-canary-interval 0`) against a mock backend: the +canary checks that invalid input is rejected with HTTP 400, which a permissive mock does not do. + +To rehearse the failure gates, degrade the backend and confirm the soak fails closed. For example, +raise the mock's latency past the request timeout and every request times out: + +```bash +MOCK_LATENCY_MS=2500 scripts/soak-rehearsal.sh --duration 30s --request-timeout 1 \ + --invalid-canary-interval 0 +# Soak FAIL: ... error_rate=100.0000% ... (error_kinds: timeout), exit 1 +``` + +## Raw throughput with vegeta + +The soak test is closed-loop and validates every response body and stream. For a quick open-loop +capacity figure — a fixed request rate, HTTP status only — [vegeta](https://github.com/tsenart/vegeta) +is a useful companion. It does not validate response bodies or streaming, so it complements the soak +rather than replacing it. + +```bash +printf 'POST http://127.0.0.1:4000/v1/chat/completions\nContent-Type: application/json\n@body.json\n' > targets.txt +printf '{"model":"RELEASE_MODEL_ID","messages":[{"role":"user","content":"ping"}],"max_tokens":8}' > body.json +vegeta attack -targets=targets.txt -rate=100 -duration=30s | vegeta report +``` + +## Run the 48-hour test + +Choose concurrency from the release capacity plan. Increase it in short +rehearsals until you find the highest expected steady load that remains below +the backend's rate limit. Use that load for the 48-hour run. An overload test +that spends most of its time throttled does not measure release stability. + +```bash +./target/release/switchyard-soak \ + --base-url http://127.0.0.1:4000 \ + --model RELEASE_MODEL_ID \ + --duration 48h \ + --concurrency 16 \ + --server-pid "$SOAK_SERVER_PID" \ + --max-rss-growth-mib 512 +``` + +The runner keeps 16 requests in flight until the test ends. This can generate +large usage charges against a metered backend. Use a dedicated test deployment, +estimate the request volume first with a short run, and get approval for any +paid-provider cost. + +Use a five-minute rehearsal to confirm the route and result files: + +```bash +./target/release/switchyard-soak \ + --base-url http://127.0.0.1:4000 \ + --model RELEASE_MODEL_ID \ + --duration 5m \ + --concurrency 4 \ + --report-interval 10 +``` + +If the Switchyard endpoint requires a bearer token, pass the environment +variable name instead of putting the token on the command line: + +```bash +export SWITCHYARD_SOAK_TOKEN="..." +./target/release/switchyard-soak \ + --api-key-env SWITCHYARD_SOAK_TOKEN \ + --model RELEASE_MODEL_ID +``` + +## Pass criteria + +The command exits with status 0 only when: + +- the requested duration completes; +- at least one inference request completes; +- the inference error rate stays at or below `--max-error-rate` (default `0`, + which means no inference request may fail); +- every periodic liveness check passes; +- every `/metrics` read returns both Switchyard request counters; +- every requested process sample returns RSS and CPU data; +- every invalid-request recovery check passes; +- the server request counter never resets; and +- RSS growth stays within `--max-rss-growth-mib` when that limit is set. + +If the release plan permits transient failures from a remote provider, set an +explicit error budget with `--max-error-rate`. Record the reason for that +exception in the release record. + +The RSS limit is deployment-specific. Set it from an approved baseline for the +same model, concurrency, and worker count. Omit `--server-pid` and +`--max-rss-growth-mib` when the server runs on another host, then collect +memory and restart data from that host's monitoring system. + +## Review the results + +Each run creates a timestamped directory under `soak-results/`: + +- `config.json` records the non-secret test inputs. +- `intervals.csv` records request rate, errors, latency percentiles, health, + Switchyard counters, RSS, and CPU once per reporting interval. `cpu_percent` + is the `ps` lifetime-average CPU for the process, not the interval's usage, so + read it as a long-run average rather than a spike detector. +- `errors.jsonl` records up to 10,000 request and canary failures. +- `summary.json` records the final pass result and any failed gates. +- `status.json` is overwritten atomically each interval with a one-object live snapshot — progress + toward the target duration, cumulative requests, cumulative error rate, health, detected restarts, + and a `status` of `ok`, `degraded`, or `stalled`. Read it (or tail the run log, whose interval line + carries the same fields and status token) to monitor a run in progress; on a remote host, + `cat /status.json` gives an at-a-glance confidence check without parsing `intervals.csv`. + +When the run ends the command also prints a latency percentile table (p50 through +p99.9) and an ASCII latency histogram to the terminal, for a quick read of the +distribution without opening the result files. + +Before approving the release, check `intervals.csv` for late failures, falling +throughput, increasing p95 or p99 latency, and steady RSS growth. Compare the +first and last several hours, not only the run-wide averages. Attach +`summary.json`, the interval chart, the Switchyard log, the tested commit, and +the route bundle to the release record. diff --git a/mkdocs.yml b/mkdocs.yml index fcf40f715..d934b61ef 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -32,6 +32,7 @@ nav: - Advisor-Gate Routing: routing_algorithms/advisor_gate_routing.md - Operations: - Context-Window Handling: operations/context_window.md + - Soak Testing: operations/soak_test.md - Reference: - CLI Reference: cli_reference.md - TOML Schema: reference/toml_schema.md diff --git a/scripts/soak-rehearsal.sh b/scripts/soak-rehearsal.sh new file mode 100755 index 000000000..8da420bb9 --- /dev/null +++ b/scripts/soak-rehearsal.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Hermetic soak rehearsal: run the soak tester against switchyard-server backed by VidaiMock +# (https://github.com/vidaiUK/VidaiMock), a mock LLM backend, with no live provider credentials +# or cost. Arguments are passed through to switchyard-soak; with none, a short rehearsal runs with +# the invalid-request canary disabled, because a permissive mock does not reject invalid input. +# +# Requires the vidaimock binary on PATH (or set VIDAIMOCK_BIN) and release builds of +# switchyard-server and switchyard-soak (cargo build --release -p switchyard-server -p switchyard-soak). +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +VIDAIMOCK_BIN="${VIDAIMOCK_BIN:-vidaimock}" +SERVER_BIN="${SWITCHYARD_SERVER_BIN:-$REPO_ROOT/target/release/switchyard-server}" +SOAK_BIN="${SWITCHYARD_SOAK_BIN:-$REPO_ROOT/target/release/switchyard-soak}" +MOCK_PORT="${MOCK_PORT:-8100}" +SERVER_PORT="${SERVER_PORT:-4000}" +MOCK_LATENCY_MS="${MOCK_LATENCY_MS:-40}" + +command -v "$VIDAIMOCK_BIN" >/dev/null 2>&1 || { + echo "error: '$VIDAIMOCK_BIN' not found on PATH." >&2 + echo "Install VidaiMock from https://github.com/vidaiUK/VidaiMock/releases, or set VIDAIMOCK_BIN." >&2 + exit 1 +} +for bin in "$SERVER_BIN" "$SOAK_BIN"; do + [ -x "$bin" ] || { + echo "error: '$bin' not found; run: cargo build --release -p switchyard-server -p switchyard-soak" >&2 + exit 1 + } +done + +workdir="$(mktemp -d)" +mock_pid="" +server_pid="" +cleanup() { + [ -n "$server_pid" ] && kill "$server_pid" 2>/dev/null || true + [ -n "$mock_pid" ] && kill "$mock_pid" 2>/dev/null || true + rm -rf "$workdir" +} +trap cleanup EXIT + +wait_health() { # $1=url $2=name + for _ in $(seq 1 60); do + curl -sf "$1" >/dev/null 2>&1 && return 0 + sleep 0.25 + done + echo "error: $2 did not become healthy at $1" >&2 + return 1 +} + +cat >"$workdir/mock.toml" <"$workdir/vidaimock.log" 2>&1 & +mock_pid=$! +disown 2>/dev/null || true # drop job control so cleanup does not print "Terminated" +wait_health "http://127.0.0.1:${MOCK_PORT}/health" "VidaiMock" + +echo "Starting switchyard-server on :${SERVER_PORT}" +"$SERVER_BIN" --config "$workdir/mock.toml" --port "$SERVER_PORT" \ + >"$workdir/switchyard-server.log" 2>&1 & +server_pid=$! +disown 2>/dev/null || true +wait_health "http://127.0.0.1:${SERVER_PORT}/health" "switchyard-server" + +if [ "$#" -gt 0 ]; then + "$SOAK_BIN" --base-url "http://127.0.0.1:${SERVER_PORT}" --model switchyard/mock "$@" +else + "$SOAK_BIN" --base-url "http://127.0.0.1:${SERVER_PORT}" --model switchyard/mock \ + --duration 60s --concurrency 8 --stream-ratio 0.5 --report-interval 10 \ + --invalid-canary-interval 0 --server-pid "$server_pid" +fi From 4b8cc61b53406fd874a1a347712ff9b12edd9574 Mon Sep 17 00:00:00 2001 From: Elyas Mehtabuddin Date: Mon, 17 Aug 2026 09:17:52 -0700 Subject: [PATCH 2/9] feat(operations): simplify release soak workflow Signed-off-by: Elyas Mehtabuddin --- Cargo.lock | 1017 ++++++++++++++++- crates/libsy/src/core/algorithm.rs | 4 + crates/switchyard-server/src/config.rs | 15 + crates/switchyard-soak/Cargo.toml | 8 + crates/switchyard-soak/README.md | 120 ++ .../switchyard-soak/examples/mock_server.rs | 58 + crates/switchyard-soak/src/client.rs | 189 ++- crates/switchyard-soak/src/lib.rs | 251 ++-- crates/switchyard-soak/src/report.rs | 200 ++-- crates/switchyard-soak/src/stats.rs | 399 +++---- crates/switchyard-soak/tests/soak.rs | 381 +++--- docs/operations/soak_test.md | 97 +- scripts/soak-rehearsal.sh | 87 -- scripts/soak_rehearsal.py | 481 ++++++++ scripts/soak_rehearsal.toml | 57 + 15 files changed, 2378 insertions(+), 986 deletions(-) create mode 100644 crates/switchyard-soak/README.md create mode 100644 crates/switchyard-soak/examples/mock_server.rs delete mode 100755 scripts/soak-rehearsal.sh create mode 100755 scripts/soak_rehearsal.py create mode 100644 scripts/soak_rehearsal.toml diff --git a/Cargo.lock b/Cargo.lock index 41f24d376..0368e0ebb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -31,6 +31,15 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" +[[package]] +name = "android_system_properties" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" +dependencies = [ + "libc", +] + [[package]] name = "anstream" version = "1.0.0" @@ -96,6 +105,12 @@ dependencies = [ "rustversion", ] +[[package]] +name = "arraydeque" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d902e3d592a523def97af8f317b08ce16b7ab854c1985a0c671e6f15cebc236" + [[package]] name = "assert-json-diff" version = "2.0.2" @@ -274,6 +289,18 @@ name = "bitflags" version = "2.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" +dependencies = [ + "serde_core", +] + +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] [[package]] name = "borrow-or-share" @@ -281,6 +308,16 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc0b364ead1874514c8c2855ab558056ebfeb775653e7ae45ff72f28f8f3166c" +[[package]] +name = "bstr" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bb31b46c14244e20ee9984b11bf5c992b91fb6939fea616e3512c8baecdbe5f" +dependencies = [ + "memchr", + "serde_core", +] + [[package]] name = "bumpalo" version = "3.20.3" @@ -334,6 +371,41 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "chrono-tz" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93698b29de5e97ad0ae26447b344c482a7284c737d9ddc5f9e52b74a336671bb" +dependencies = [ + "chrono", + "chrono-tz-build", + "phf", +] + +[[package]] +name = "chrono-tz-build" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c088aee841df9c3041febbb73934cfc39708749bf96dc827e3359cd39ef11b1" +dependencies = [ + "parse-zoneinfo", + "phf", + "phf_codegen", +] + [[package]] name = "clap" version = "4.6.2" @@ -399,6 +471,61 @@ dependencies = [ "memchr", ] +[[package]] +name = "config" +version = "0.15.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b85f248a4de22d204ceabc6299d89d2c70fbd7f09fea53c06c852369652d8139" +dependencies = [ + "async-trait", + "convert_case", + "json5", + "pathdiff", + "ron", + "rust-ini", + "serde-untagged", + "serde_core", + "serde_json", + "toml", + "winnow", + "yaml-rust2", +] + +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + +[[package]] +name = "const-random" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom 0.2.17", + "once_cell", + "tiny-keccak", +] + +[[package]] +name = "convert_case" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec182b0ca2f35d8fc196cf3404988fd8b8c739a4d270ff118a398feb0cbec1ca" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "core-foundation" version = "0.10.1" @@ -424,6 +551,64 @@ dependencies = [ "libc", ] +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d85363c37faeca707aef026efa9f3b34d077bce547e48f770770625c6013679e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + [[package]] name = "data-encoding" version = "2.11.1" @@ -448,12 +633,35 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "092966b41edc516079bdf31ec78a2e0588d1d0c08f78b91d8307215928642b2b" +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" + +[[package]] +name = "deunicode" +version = "1.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abd57806937c9cc163efc8ea3910e00a62e2aeb0b8119f1793a978088f8f6b04" + [[package]] name = "diff" version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer", + "const-oid", + "crypto-common", +] + [[package]] name = "displaydoc" version = "0.2.6" @@ -465,6 +673,15 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "dlv-list" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "442039f5147480ba31067cb00ada1adae6892028e40e45fc5de7b7df6dcc1b5f" +dependencies = [ + "const-random", +] + [[package]] name = "dunce" version = "1.0.5" @@ -486,12 +703,32 @@ dependencies = [ "serde", ] +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + [[package]] name = "equivalent" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +[[package]] +name = "erased-serde" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2add8a07dd6a8d93ff627029c51de145e12686fbc36ecb298ac22e74cf02dec" +dependencies = [ + "serde", + "serde_core", + "typeid", +] + [[package]] name = "errno" version = "0.3.14" @@ -712,6 +949,36 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "glob" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" + +[[package]] +name = "globset" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e47d37d2ae4464254884b60ab7071be2b876a9c35b696bd018ddcc76847309cd" +dependencies = [ + "aho-corasick", + "bstr", + "log", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "globwalk" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf760ebf69878d9fd8f110c89703d90ce35095324d1f1edcb595c63945ee757" +dependencies = [ + "bitflags", + "ignore", + "walkdir", +] + [[package]] name = "h2" version = "0.4.15" @@ -731,6 +998,24 @@ dependencies = [ "tracing", ] +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "foldhash", +] + [[package]] name = "hashbrown" version = "0.17.1" @@ -742,6 +1027,15 @@ dependencies = [ "foldhash", ] +[[package]] +name = "hashlink" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824e001ac4f3012dd16a264bec811403a67ca9deb6c102fc5049b32c4574b35f" +dependencies = [ + "hashbrown 0.16.1", +] + [[package]] name = "heck" version = "0.5.0" @@ -799,12 +1093,30 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "humansize" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cb51c9a029ddc91b07a787f1d86b53ccfa49b0e86688c946ebe8d3555685dd7" +dependencies = [ + "libm", +] + [[package]] name = "humantime" version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15" +[[package]] +name = "hybrid-array" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "707114b52a152fa7bdb290cd7cd5912d9467273b6d74e21b8d81aca1f8533f6b" +dependencies = [ + "typenum", +] + [[package]] name = "hyper" version = "1.10.1" @@ -836,7 +1148,9 @@ dependencies = [ "http", "hyper", "hyper-util", + "log", "rustls", + "rustls-native-certs", "tokio", "tokio-rustls", "tower-service", @@ -865,6 +1179,30 @@ dependencies = [ "tracing", ] +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + [[package]] name = "icu_collections" version = "2.2.0" @@ -968,6 +1306,22 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "ignore" +version = "0.4.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b17771570a2b94107741a7b033f19132c2eee21d59d21b24d2ced26500bd66e" +dependencies = [ + "crossbeam-deque", + "globset", + "log", + "memchr", + "regex-automata", + "same-file", + "walkdir", + "winapi-util", +] + [[package]] name = "indexmap" version = "2.14.0" @@ -975,7 +1329,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", - "hashbrown", + "hashbrown 0.17.1", ] [[package]] @@ -1075,6 +1429,17 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "json5" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96b0db21af676c1ce64250b5f40f3ce2cf27e4e47cb91ed91eb6fe9350b430c1" +dependencies = [ + "pest", + "pest_derive", + "serde", +] + [[package]] name = "jsonptr" version = "0.8.1" @@ -1149,6 +1514,21 @@ version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "libmimalloc-sys" +version = "0.1.49" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a45a52f43e1c16f667ccfe4dd8c85b7f7c204fd5e3bf46c5b0db9a5c3c0b8e9" +dependencies = [ + "cc", +] + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -1203,18 +1583,83 @@ version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" +[[package]] +name = "metrics" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3045b4193fbdc5b5681f32f11070da9be3609f189a79f3390706d42587f46bb5" +dependencies = [ + "ahash", + "portable-atomic", +] + +[[package]] +name = "metrics-exporter-prometheus" +version = "0.15.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4f0c8427b39666bf970460908b213ec09b3b350f20c0c2eabcbba51704a08e6" +dependencies = [ + "base64", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "indexmap", + "ipnet", + "metrics", + "metrics-util", + "quanta", + "thiserror 1.0.69", + "tokio", + "tracing", +] + +[[package]] +name = "metrics-util" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4259040465c955f9f2f1a4a8a16dc46726169bca0f88e8fb2dbeced487c3e828" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", + "hashbrown 0.14.5", + "metrics", + "num_cpus", + "quanta", + "sketches-ddsketch", +] + [[package]] name = "micromap" version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a86d3146ed3995b5913c414f6664344b9617457320782e64f0bb44afd49d74" +[[package]] +name = "mimalloc" +version = "0.1.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d4139bb28d14ad1facf21d5eb8825051b326e172d216b39f6d31df53cc97862" +dependencies = [ + "libmimalloc-sys", +] + [[package]] name = "mime" version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "mime_guess" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e" +dependencies = [ + "mime", + "unicase", +] + [[package]] name = "mio" version = "1.2.2" @@ -1274,6 +1719,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + [[package]] name = "num-integer" version = "0.1.46" @@ -1425,6 +1876,16 @@ dependencies = [ "tokio", ] +[[package]] +name = "ordered-multimap" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49203cdcae0030493bad186b28da2fa25645fa276a51b6fec8010d281e02ef79" +dependencies = [ + "dlv-list", + "hashbrown 0.14.5", +] + [[package]] name = "outref" version = "0.5.2" @@ -1454,12 +1915,107 @@ dependencies = [ "windows-link", ] +[[package]] +name = "parse-zoneinfo" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f2a05b18d44e2957b88f96ba460715e295bc1d7510468a2f3d3b44535d26c24" +dependencies = [ + "regex", +] + +[[package]] +name = "pathdiff" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" + [[package]] name = "percent-encoding" version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "pest" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a07a60cc7a4d00c91f95c685609d1d2f79050e6804b70ebedd7650f0b839bcf" +dependencies = [ + "memchr", + "ucd-trie", +] + +[[package]] +name = "pest_derive" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3a83744a5c8455b8b3e0dc5031362780a347c878bdd11584d1a8984228cc88d" +dependencies = [ + "pest", + "pest_generator", +] + +[[package]] +name = "pest_generator" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0cd3451aa3de60d4b9a1e736885e4dea6b31617598026f12256ad566d63304a" +dependencies = [ + "pest", + "pest_meta", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "pest_meta" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e04d3a0849e241d7dfce834c83b1c5edc8622009e8dd51a12ba1927c32f05496" +dependencies = [ + "pest", +] + +[[package]] +name = "phf" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" +dependencies = [ + "phf_shared", +] + +[[package]] +name = "phf_codegen" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a" +dependencies = [ + "phf_generator", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" +dependencies = [ + "phf_shared", + "rand 0.8.7", +] + +[[package]] +name = "phf_shared" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" +dependencies = [ + "siphasher", +] + [[package]] name = "pin-project-lite" version = "0.2.17" @@ -1487,6 +2043,12 @@ dependencies = [ "zerovec", ] +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -1655,6 +2217,21 @@ dependencies = [ "serde", ] +[[package]] +name = "quanta" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3ab5a9d756f0d97bdc89019bd2e4ea098cf9cde50ee7564dde6b81ccc8f06c7" +dependencies = [ + "crossbeam-utils", + "libc", + "once_cell", + "raw-cpuid", + "wasi", + "web-sys", + "winapi", +] + [[package]] name = "quinn" version = "0.11.11" @@ -1733,13 +2310,24 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +[[package]] +name = "rand" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + [[package]] name = "rand" version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" dependencies = [ - "rand_chacha", + "rand_chacha 0.9.0", "rand_core 0.9.5", ] @@ -1747,11 +2335,21 @@ dependencies = [ name = "rand" version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core 0.10.1", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ - "chacha20", - "getrandom 0.4.3", - "rand_core 0.10.1", + "ppv-lite86", + "rand_core 0.6.4", ] [[package]] @@ -1764,6 +2362,15 @@ dependencies = [ "rand_core 0.9.5", ] +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + [[package]] name = "rand_core" version = "0.9.5" @@ -1788,6 +2395,15 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "raw-cpuid" +version = "11.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" +dependencies = [ + "bitflags", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -1826,7 +2442,7 @@ dependencies = [ "ahash", "fluent-uri", "getrandom 0.3.4", - "hashbrown", + "hashbrown 0.17.1", "itoa", "micromap", "parking_lot", @@ -1918,6 +2534,65 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "ron" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81116b9531d61eabc41aeb228e4b6b2435bcca3233b98cf3b3077d4e6e9debb3" +dependencies = [ + "bitflags", + "once_cell", + "serde", + "serde_derive", + "typeid", + "unicode-ident", +] + +[[package]] +name = "rust-embed" +version = "8.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9e7760e252aaba7b09f4be00e36476cf585bdb68a53552ac954cdf504ab4bc9" +dependencies = [ + "rust-embed-impl", + "rust-embed-utils", + "walkdir", +] + +[[package]] +name = "rust-embed-impl" +version = "8.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bcfc4d6f53af43755f7a723e4b6b8794fcce052a178dd8c6c1dadc5f5343097" +dependencies = [ + "mime_guess", + "proc-macro2", + "quote", + "rust-embed-utils", + "syn 2.0.119", + "walkdir", +] + +[[package]] +name = "rust-embed-utils" +version = "8.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42ffa149f6aa81b58a5b3011d01a857c4ed12c7a732d2c51947a4c7c692185f0" +dependencies = [ + "sha2", + "walkdir", +] + +[[package]] +name = "rust-ini" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "796e8d2b6696392a43bea58116b667fb4c29727dc5abd27d6acf338bb4f688c7" +dependencies = [ + "cfg-if", + "ordered-multimap", +] + [[package]] name = "rustc-hash" version = "2.1.3" @@ -1953,6 +2628,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c54fcab019b409d04215d3a17cb438fd7fbf192ee61461f20f4fe18704bc138" dependencies = [ "aws-lc-rs", + "log", "once_cell", "rustls-pki-types", "rustls-webpki", @@ -2096,6 +2772,18 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde-untagged" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9faf48a4a2d2693be24c6289dbe26552776eb7737074e6722891fadbe6c5058" +dependencies = [ + "erased-serde", + "serde", + "serde_core", + "typeid", +] + [[package]] name = "serde_core" version = "1.0.228" @@ -2161,6 +2849,30 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_yaml" +version = "0.9.34+deprecated" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" +dependencies = [ + "indexmap", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", +] + +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "sharded-slab" version = "0.1.7" @@ -2202,12 +2914,34 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" +[[package]] +name = "siphasher" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" + +[[package]] +name = "sketches-ddsketch" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85636c14b73d81f541e525f585c0a2109e6744e1565b5c1668e31c70c10ed65c" + [[package]] name = "slab" version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" +[[package]] +name = "slug" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "882a80f72ee45de3cc9a5afeb2da0331d58df69e4e7d8eeb5d3c7784ae67e724" +dependencies = [ + "deunicode", + "wasm-bindgen", +] + [[package]] name = "smallvec" version = "1.15.2" @@ -2395,12 +3129,15 @@ dependencies = [ "axum", "clap", "futures-util", + "humantime", "parking_lot", "rand 0.10.2", "reqwest", + "serde", "serde_json", "tempfile", "tokio", + "vidaimock", ] [[package]] @@ -2416,6 +3153,12 @@ dependencies = [ "thiserror 2.0.18", ] +[[package]] +name = "symlink" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7973cce6668464ea31f176d85b13c7ab3bba2cb3b77a2ed26abd7801688010a" + [[package]] name = "syn" version = "2.0.119" @@ -2477,6 +3220,28 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "tera" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8004bca281f2d32df3bacd59bc67b312cb4c70cea46cbd79dbe8ac5ed206722" +dependencies = [ + "chrono", + "chrono-tz", + "globwalk", + "humansize", + "lazy_static", + "percent-encoding", + "pest", + "pest_derive", + "rand 0.8.7", + "regex", + "serde", + "serde_json", + "slug", + "unicode-segmentation", +] + [[package]] name = "thiserror" version = "1.0.69" @@ -2526,6 +3291,45 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "time" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + +[[package]] +name = "time-macros" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + [[package]] name = "tinystr" version = "0.8.3" @@ -2683,6 +3487,7 @@ dependencies = [ "tower", "tower-layer", "tower-service", + "tracing", "url", ] @@ -2710,6 +3515,19 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-appender" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "050686193eb999b4bb3bc2acfa891a13da00f79734704c4b8b4ef1a10b368a3c" +dependencies = [ + "crossbeam-channel", + "symlink", + "thiserror 2.0.18", + "time", + "tracing-subscriber", +] + [[package]] name = "tracing-attributes" version = "0.1.31" @@ -2758,6 +3576,16 @@ dependencies = [ "web-time", ] +[[package]] +name = "tracing-serde" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704b1aeb7be0d0a84fc9828cae51dab5970fee5088f83d1dd7ee6f6246fc6ff1" +dependencies = [ + "serde", + "tracing-core", +] + [[package]] name = "tracing-subscriber" version = "0.3.23" @@ -2768,12 +3596,15 @@ dependencies = [ "nu-ansi-term", "once_cell", "regex-automata", + "serde", + "serde_json", "sharded-slab", "smallvec", "thread_local", "tracing", "tracing-core", "tracing-log", + "tracing-serde", ] [[package]] @@ -2782,6 +3613,30 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "typeid" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c" + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "ucd-trie" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" + +[[package]] +name = "unicase" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" + [[package]] name = "unicode-general-category" version = "1.1.0" @@ -2794,6 +3649,18 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + +[[package]] +name = "unsafe-libyaml" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" + [[package]] name = "untrusted" version = "0.9.0" @@ -2824,6 +3691,17 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "uuid" +version = "1.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cefc03fd367c0c6d4305de1b312cf00248c4114f4a0418ce6a6af769e3b0bd9" +dependencies = [ + "getrandom 0.4.3", + "js-sys", + "wasm-bindgen", +] + [[package]] name = "uuid-simd" version = "0.8.0" @@ -2846,6 +3724,42 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "vidaimock" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8887e1668af80f23c5ee7b774f75e76439d5f9d91ba80784bac6b78bb73d3fe8" +dependencies = [ + "axum", + "base64", + "chrono", + "clap", + "config", + "crc32fast", + "futures", + "glob", + "metrics", + "metrics-exporter-prometheus", + "mimalloc", + "num_cpus", + "once_cell", + "rand 0.9.5", + "regex", + "rust-embed", + "serde", + "serde_json", + "serde_yaml", + "tera", + "tokio", + "tower", + "tower-http", + "tracing", + "tracing-appender", + "tracing-subscriber", + "uuid", + "walkdir", +] + [[package]] name = "vsimd" version = "0.8.0" @@ -2983,6 +3897,22 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + [[package]] name = "winapi-util" version = "0.1.11" @@ -2992,12 +3922,71 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "windows-link" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + [[package]] name = "windows-sys" version = "0.52.0" @@ -3085,6 +4074,9 @@ name = "winnow" version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" +dependencies = [ + "memchr", +] [[package]] name = "wiremock" @@ -3121,6 +4113,17 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" +[[package]] +name = "yaml-rust2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "631a50d867fafb7093e709d75aaee9e0e0d5deb934021fcea25ac2fe09edc51e" +dependencies = [ + "arraydeque", + "encoding_rs", + "hashlink", +] + [[package]] name = "yansi" version = "1.0.1" diff --git a/crates/libsy/src/core/algorithm.rs b/crates/libsy/src/core/algorithm.rs index 47ddfb668..13f1cd441 100644 --- a/crates/libsy/src/core/algorithm.rs +++ b/crates/libsy/src/core/algorithm.rs @@ -422,6 +422,10 @@ mod tests { LibsyError::external("test", TestError(message)) } + /// Build a routed decision for orchestration tests. + fn test_decision(selected_model_id: String) -> Decision { + Decision::new(selected_model_id, None, true) + } /// Trivial algo used only to exercise the orchestrator: calls the first target /// and returns its response as the routing outcome. struct TestAlgo { diff --git a/crates/switchyard-server/src/config.rs b/crates/switchyard-server/src/config.rs index a78695cd5..ea5e86d43 100644 --- a/crates/switchyard-server/src/config.rs +++ b/crates/switchyard-server/src/config.rs @@ -1652,6 +1652,21 @@ target = "azure" return Err(ServerError::new("primary llm client is missing")); }; assert_eq!(primary.max_retries, 0); + + let maximum = VALID_CONFIG.replacen( + "base_url = \"https://example.test/v1\"", + &format!( + "base_url = \"https://example.test/v1\"\nmax_retries = {MAX_CONFIGURED_RETRIES}" + ), + 1, + ); + let config: ServerConfig = toml::from_str(&maximum).map_err(|error| { + ServerError::new(format!("failed to parse maximum retry config: {error}")) + })?; + let Some(primary) = config.llm_clients.get("primary") else { + return Err(ServerError::new("primary llm client is missing")); + }; + assert_eq!(primary.max_retries, MAX_CONFIGURED_RETRIES); Ok(()) } diff --git a/crates/switchyard-soak/Cargo.toml b/crates/switchyard-soak/Cargo.toml index 771cd78be..c2b449c32 100644 --- a/crates/switchyard-soak/Cargo.toml +++ b/crates/switchyard-soak/Cargo.toml @@ -10,17 +10,24 @@ edition.workspace = true license.workspace = true repository.workspace = true rust-version.workspace = true +readme = "README.md" [[bin]] name = "switchyard-soak" path = "src/main.rs" +[[example]] +name = "switchyard-soak-mock" +path = "examples/mock_server.rs" + [dependencies] clap = { version = "4", features = ["derive"] } futures-util.workspace = true +humantime = "2.4" parking_lot.workspace = true rand.workspace = true reqwest.workspace = true +serde.workspace = true serde_json.workspace = true tokio.workspace = true @@ -28,3 +35,4 @@ tokio.workspace = true axum = "0.8" tempfile = "3" tokio.workspace = true +vidaimock = "0.3" diff --git a/crates/switchyard-soak/README.md b/crates/switchyard-soak/README.md new file mode 100644 index 000000000..0134236c1 --- /dev/null +++ b/crates/switchyard-soak/README.md @@ -0,0 +1,120 @@ +# switchyard-soak + +`switchyard-soak` keeps a fixed number of requests in flight against one route on a live +`switchyard-server`. It exercises Chat Completions, Messages, and Responses in both streaming and +non-streaming form. It also samples health, metrics, and an optional local server process. The +command writes evidence to a new results directory and exits with status 1 when a release gate +fails. + +Build the server and soak tester from the commit being tested: + +```bash +cargo build --release -p switchyard-server -p switchyard-soak +``` + +Run a five-minute check against a route advertised by `GET /v1/models`: + +```bash +target/release/switchyard-soak \ + --base-url http://127.0.0.1:4000 \ + --model switchyard/general \ + --duration 5m \ + --concurrency 4 \ + --report-interval 10 +``` + +Run the 48-hour release gate and measure the local server process: + +```bash +target/release/switchyard-soak \ + --model switchyard/general \ + --duration 48h \ + --concurrency 16 \ + --server-pid "$SWITCHYARD_SERVER_PID" \ + --max-rss-growth-mib 512 +``` + +If the endpoint needs a bearer token, put the secret in an environment variable and pass the +variable's name: + +```bash +export SWITCHYARD_SOAK_TOKEN="..." +target/release/switchyard-soak \ + --model switchyard/general \ + --api-key-env SWITCHYARD_SOAK_TOKEN +``` + +## Flags + +| Flag | Default | What it does | +|---|---:|---| +| `--base-url URL` | `http://127.0.0.1:4000` | Sends every check and inference request to this Switchyard server. | +| `--model ID` | required | Selects one exact route id returned by `GET /v1/models`. The test stops before sending load if the id is missing. | +| `--duration TIME` | `48h` | Keeps the load running for this long. Use seconds, minutes, or hours, such as `30s`, `5m`, or `48h`. | +| `--concurrency N` | `16` | Keeps this many requests in flight. Each worker waits for its response before sending another request. | +| `--max-output-tokens N` | `32` | Sends this output-token limit with every request. It must be at least 1. | +| `--prompt-bytes N` | `1024` | Adds this many bytes of repeated prefix to each of four prompts. Larger values put more pressure on request memory and prefix caching. | +| `--request-timeout SECONDS` | `120` | Limits connection time and idle time between response bytes. A healthy stream may run longer if it keeps producing data. | +| `--report-interval SECONDS` | `60` | Sets how often the test samples health, metrics, RSS, CPU, and interval latency. | +| `--invalid-canary-interval SECONDS` | `300` | Sends malformed input this often, expects HTTP 400, and then checks recovery. Set it to `0` only when a permissive mock cannot reject the canary. | +| `--max-error-rate FRACTION` | `0` | Allows at most this fraction of inference requests to fail. `0.01` means one percent. The value must be between 0 and 1. | +| `--server-pid PID` | none | Samples RSS and CPU from this local `switchyard-server` process. Omit it for a remote server. | +| `--max-rss-growth-mib MIB` | none | Fails when the last RSS sample exceeds the first by more than this amount. It requires `--server-pid`. | +| `--api-key-env NAME` | none | Reads a bearer token from this environment variable without putting the secret in the command or result files. | +| `--results-dir PATH` | `soak-results/` | Creates this new directory for the run. The command refuses to reuse an existing directory. | +| `--help` | n/a | Prints the command reference and examples. | +| `--version` | n/a | Prints the crate version. | + +The inference workers use a fixed six-case cycle: three public endpoint formats multiplied by +streaming on and off. A deterministic cycle makes missing endpoint or streaming coverage visible +in short runs. + +## Rehearse the whole toolchain + +The repository's `scripts/soak_rehearsal.py` starts an embedded VidaiMock server and +`switchyard-server`, validates a configuration containing every server route type, sends one +production HTTP request through each route, then runs three complementary clients: + +- `oha` checks raw HTTP concurrency and status-code distribution. +- NVIDIA AIPerf measures LLM request, token, and streaming latency through passthrough routing. +- `switchyard-soak` checks all three API formats, streaming framing, liveness, metrics, process + sampling, result files, and pass/fail gates through the same passthrough route. + +`switchyard-soak` itself does not require VidaiMock, oha, or AIPerf. The local rehearsal builds +VidaiMock's Rust library into a helper command, so it does not need a separate `vidaimock` install. +Build the Rust commands and helper: + +```bash +cargo build --release -p switchyard-server -p switchyard-soak \ + --bins --example switchyard-soak-mock +``` + +Install the two external load generators: + +```bash +cargo install oha +uv tool install aiperf +``` + +Then run the local rehearsal: + +```bash +python3.12 scripts/soak_rehearsal.py --duration 10s --concurrency 4 +``` + +The script checks every required command before it starts a server. When a command is missing, it +prints a warning with the build or install command and the environment variable that can point to +an existing executable. Cargo dependencies compile Rust libraries; a normal `cargo build` does +not install unrelated host commands, so oha and AIPerf remain explicit installs. + +The embedded VidaiMock server supplies deterministic local model responses, so this rehearsal needs no provider key +and incurs no inference cost. It is a preflight check, not a replacement for the 48-hour run +against the release deployment. + +## Results + +Each run creates `config.json`, `intervals.csv`, `errors.jsonl`, and `summary.json`. Error details +stop at 10,000 records, and run-wide latency uses a 100,000-sample reservoir, so failures and a +high request rate cannot grow memory or disk use without bound. + +See `docs/operations/soak_test.md` for release preparation, pass criteria, and result review. diff --git a/crates/switchyard-soak/examples/mock_server.rs b/crates/switchyard-soak/examples/mock_server.rs new file mode 100644 index 000000000..36d712505 --- /dev/null +++ b/crates/switchyard-soak/examples/mock_server.rs @@ -0,0 +1,58 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Local VidaiMock server used by `scripts/soak_rehearsal.py`. + +use std::process::ExitCode; + +use clap::Parser; +use vidaimock::MockServer; + +/// Start the embedded mock backend used by the local soak rehearsal. +#[derive(Parser)] +#[command( + name = "switchyard-soak-mock", + about = "Start VidaiMock for the local Switchyard soak rehearsal", + after_long_help = "Example:\n cargo run --release -p switchyard-soak --example switchyard-soak-mock -- --port 8100 --latency-ms 40", + version +)] +struct Args { + /// Local TCP port used by the mock backend. + #[arg(long, default_value_t = 8100)] + port: u16, + + /// Artificial delay, in milliseconds, added to every mock response. + #[arg(long, default_value_t = 40)] + latency_ms: u64, +} + +async fn run(args: Args) -> Result<(), String> { + if args.port == 0 { + return Err("--port must be greater than zero".to_string()); + } + + let server = MockServer::builder() + .bind(format!("127.0.0.1:{}", args.port)) + .mode("realistic") + .latency_ms(args.latency_ms) + .start() + .await + .map_err(|error| error.to_string())?; + + println!("VidaiMock is ready at {}", server.base_url()); + tokio::signal::ctrl_c() + .await + .map_err(|error| format!("could not wait for shutdown: {error}"))?; + server.shutdown().await.map_err(|error| error.to_string()) +} + +#[tokio::main] +async fn main() -> ExitCode { + match run(Args::parse()).await { + Ok(()) => ExitCode::SUCCESS, + Err(error) => { + eprintln!("switchyard-soak-mock failed: {error}"); + ExitCode::FAILURE + } + } +} diff --git a/crates/switchyard-soak/src/client.rs b/crates/switchyard-soak/src/client.rs index e3f3285f5..e8fd7a7ed 100644 --- a/crates/switchyard-soak/src/client.rs +++ b/crates/switchyard-soak/src/client.rs @@ -3,18 +3,16 @@ //! HTTP against the public Switchyard APIs: preflight, one request, and server-state reads. -use std::collections::BTreeMap; - -use clap::ValueEnum; use futures_util::StreamExt; use reqwest::Client; use reqwest::header::CONTENT_TYPE; use serde_json::{Value, json}; -use crate::stats::{SERVER_ERRORS_METRIC, SERVER_REQUESTS_METRIC}; +const SERVER_REQUESTS_METRIC: &str = "switchyard_total_requests"; +const SERVER_ERRORS_METRIC: &str = "switchyard_total_errors"; /// One public Switchyard API the soak test exercises. -#[derive(Clone, Copy, PartialEq, Eq, ValueEnum)] +#[derive(Clone, Copy, PartialEq, Eq)] pub enum Endpoint { Chat, Messages, @@ -55,15 +53,12 @@ fn truncate(text: &str, limit: usize) -> String { text.chars().take(limit).collect() } -/// Map a transport-level reqwest error to a short, stable error kind. fn transport_error_kind(error: &reqwest::Error) -> &'static str { if error.is_timeout() { "timeout" } else if error.is_decode() { - // A body decode failure mirrors httpx's DecodingError, which reported "request_error". "request_error" } else if error.is_connect() || error.is_request() || error.is_body() { - // Connect, send, and mid-stream read failures are httpx TransportError -> "transport". "transport" } else { "request_error" @@ -96,10 +91,14 @@ pub fn request_body( } } -/// Read the process-wide Switchyard counters from Prometheus text. -pub fn parse_metrics(text: &str) -> BTreeMap { - let wanted = [SERVER_REQUESTS_METRIC, SERVER_ERRORS_METRIC]; - let mut parsed = BTreeMap::new(); +#[derive(Default)] +struct Metrics { + requests: Option, + errors: Option, +} + +fn parse_metrics(text: &str) -> Metrics { + let mut metrics = Metrics::default(); for line in text.lines() { if line.is_empty() || line.starts_with('#') { continue; @@ -108,45 +107,62 @@ pub fn parse_metrics(text: &str) -> BTreeMap { continue; }; let name = name_part.split('{').next().unwrap_or(name_part); - if !wanted.contains(&name) { - continue; - } if let Some(token) = rest.split_whitespace().next() && let Ok(value) = token.parse::() { - parsed.insert(name.to_string(), value); + match name { + SERVER_REQUESTS_METRIC => metrics.requests = Some(value), + SERVER_ERRORS_METRIC => metrics.errors = Some(value), + _ => {} + } } } - parsed + metrics +} + +pub struct RequestError { + pub kind: String, + pub detail: String, +} + +impl RequestError { + fn new(kind: impl Into, detail: impl Into) -> Self { + Self { + kind: kind.into(), + detail: detail.into(), + } + } + + fn transport(error: &reqwest::Error) -> Self { + Self::new( + transport_error_kind(error), + truncate(&error.to_string(), 500), + ) + } } -/// Send one request and return an error kind and detail, if any. +/// Send one request and validate its response body or event stream. pub async fn send_request( client: &Client, base_url: &str, endpoint: Endpoint, body: &Value, -) -> (Option, Option) { +) -> Result<(), RequestError> { let url = format!("{base_url}{}", endpoint.path()); let stream = body.get("stream").and_then(Value::as_bool).unwrap_or(false); let response = match client.post(&url).json(body).send().await { Ok(response) => response, - Err(error) => { - return ( - Some(transport_error_kind(&error).to_string()), - Some(truncate(&error.to_string(), 500)), - ); - } + Err(error) => return Err(RequestError::transport(&error)), }; if stream { let status = response.status(); if !status.is_success() { let content = response.text().await.unwrap_or_default(); - return ( - Some(format!("http_{}", status.as_u16())), - Some(truncate(&content, 500)), - ); + return Err(RequestError::new( + format!("http_{}", status.as_u16()), + truncate(&content, 500), + )); } let content_type = response .headers() @@ -156,13 +172,13 @@ pub async fn send_request( .to_string(); if !content_type.contains("text/event-stream") { let content = response.text().await.unwrap_or_default(); - return ( - Some("invalid_stream".to_string()), - Some(format!( + return Err(RequestError::new( + "invalid_stream", + format!( "expected text/event-stream, received {content_type:?}: {}", truncate(&content, 300) - )), - ); + ), + )); } let mut bytes = response.bytes_stream(); let mut received_data = false; @@ -173,69 +189,54 @@ pub async fn send_request( received_data || chunk.iter().any(|byte| !byte.is_ascii_whitespace()); } Err(error) => { - return ( - Some(transport_error_kind(&error).to_string()), - Some(truncate(&error.to_string(), 500)), - ); + return Err(RequestError::transport(&error)); } } } if !received_data { - return ( - Some("empty_stream".to_string()), - Some("successful streaming response contained no data".to_string()), - ); + return Err(RequestError::new( + "empty_stream", + "successful streaming response contained no data", + )); } - return (None, None); + return Ok(()); } let status = response.status(); if !status.is_success() { let content = response.text().await.unwrap_or_default(); - return ( - Some(format!("http_{}", status.as_u16())), - Some(truncate(&content, 500)), - ); + return Err(RequestError::new( + format!("http_{}", status.as_u16()), + truncate(&content, 500), + )); } let text = match response.text().await { Ok(text) => text, - Err(error) => { - return ( - Some(transport_error_kind(&error).to_string()), - Some(truncate(&error.to_string(), 500)), - ); - } + Err(error) => return Err(RequestError::transport(&error)), }; let payload: Value = match serde_json::from_str(&text) { Ok(payload) => payload, - Err(error) => return (Some("invalid_json".to_string()), Some(error.to_string())), + Err(error) => return Err(RequestError::new("invalid_json", error.to_string())), }; if !payload.is_object() || payload.get("error").is_some() { - return ( - Some("invalid_response".to_string()), - Some(truncate(&text, 500)), - ); + return Err(RequestError::new("invalid_response", truncate(&text, 500))); } let field = endpoint.required_field(); if payload.get(field).is_none() { - return ( - Some("invalid_response".to_string()), - Some(format!( + return Err(RequestError::new( + "invalid_response", + format!( "successful {} response did not contain {field:?}: {}", endpoint.as_str(), truncate(&text, 300) - )), - ); + ), + )); } - (None, None) + Ok(()) } -/// Check liveness and model discovery, then return the selected model. -pub async fn preflight( - client: &Client, - base_url: &str, - requested_model: Option<&str>, -) -> Result { +/// Check liveness and verify that the requested model is advertised. +pub async fn preflight(client: &Client, base_url: &str, model: &str) -> Result<(), String> { let health = client .get(format!("{base_url}/health")) .send() @@ -277,32 +278,23 @@ pub async fn preflight( truncate(&text, 300) ) })?; - let model_ids: Vec = entries + if !entries .iter() - .filter_map(|entry| entry.get("id").and_then(Value::as_str).map(str::to_string)) - .collect(); - let default_model = body - .get("default_model") - .and_then(Value::as_str) - .filter(|id| model_ids.iter().any(|known| known == id)); - - let model = requested_model - .map(str::to_string) - .or_else(|| default_model.map(str::to_string)) - .or_else(|| model_ids.first().cloned()) - .ok_or_else(|| "GET /v1/models returned no model to test".to_string())?; - if let Some(requested) = requested_model - && !model_ids.iter().any(|known| known == requested) + .any(|entry| entry.get("id").and_then(Value::as_str) == Some(model)) { - return Err(format!( - "model {requested:?} is not listed by GET /v1/models" - )); + return Err(format!("model {model:?} is not listed by GET /v1/models")); } - Ok(model) + Ok(()) } -/// Read liveness and cumulative server metrics; never errors, so one bad read is one failed sample. -pub async fn read_server_state(client: &Client, base_url: &str) -> (bool, BTreeMap) { +pub struct ServerState { + pub healthy: bool, + pub requests: Option, + pub errors: Option, +} + +/// Read liveness and cumulative metrics; one bad read becomes one failed sample. +pub async fn read_server_state(client: &Client, base_url: &str) -> ServerState { let healthy = match client.get(format!("{base_url}/health")).send().await { Ok(response) if response.status() == reqwest::StatusCode::OK => response .text() @@ -323,9 +315,13 @@ pub async fn read_server_state(client: &Client, base_url: &str) -> (bool, BTreeM .await .map(|text| parse_metrics(&text)) .unwrap_or_default(), - _ => BTreeMap::new(), + _ => Metrics::default(), }; - (healthy, metrics) + ServerState { + healthy, + requests: metrics.requests, + errors: metrics.errors, + } } #[cfg(test)] @@ -349,7 +345,7 @@ mod tests { } #[test] - fn parse_metrics_reads_counter_names_and_ignores_labelled_series() { + fn parse_metrics_reads_only_required_counters() { let metrics = parse_metrics( "# TYPE switchyard_total_requests gauge\n\ switchyard_total_requests 42\n\ @@ -357,8 +353,7 @@ mod tests { switchyard_requests_total{model=\"route\"} 10\n", ); - assert_eq!(metrics.get("switchyard_total_requests"), Some(&42.0)); - assert_eq!(metrics.get("switchyard_total_errors"), Some(&3.0)); - assert_eq!(metrics.get("switchyard_requests_total"), None); + assert_eq!(metrics.requests, Some(42.0)); + assert_eq!(metrics.errors, Some(3.0)); } } diff --git a/crates/switchyard-soak/src/lib.rs b/crates/switchyard-soak/src/lib.rs index 2dbfe1db8..6480e5f02 100644 --- a/crates/switchyard-soak/src/lib.rs +++ b/crates/switchyard-soak/src/lib.rs @@ -1,15 +1,11 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Sustained, closed-loop load test against a live Switchyard server. -//! -//! Workers keep a fixed number of inference requests in flight while a reporter samples -//! liveness, metrics, and process resources, and a canary confirms invalid input is rejected. -//! The run writes result files and exits non-zero when a release gate fails. +#![doc = include_str!("../README.md")] -pub mod client; -pub mod report; -pub mod stats; +mod client; +mod report; +mod stats; use std::fs; use std::future::Future; @@ -21,8 +17,6 @@ use std::time::{Duration, Instant}; use clap::Parser; use parking_lot::Mutex; -use rand::rngs::StdRng; -use rand::{RngExt, SeedableRng}; use reqwest::Client; use reqwest::header::{AUTHORIZATION, HeaderMap, HeaderValue}; use serde_json::json; @@ -31,8 +25,7 @@ use tokio::sync::Notify; use crate::client::{Endpoint, preflight, request_body, send_request}; use crate::report::{ResultsWriter, invalid_request_canary, reporter}; use crate::stats::{ - RunStats, build_prompt_pool, build_summary, latency_report, now_utc_string, round3, - utc_dir_stamp, + RunStats, build_prompt_pool, build_summary, now_utc_string, round3, utc_dir_stamp, }; /// Command-line arguments for the soak test. @@ -40,102 +33,88 @@ use crate::stats::{ #[command( name = "switchyard-soak", about = "Run a sustained, closed-loop load test against a live Switchyard server", + after_long_help = "Examples:\n switchyard-soak --model switchyard/general --duration 5m --concurrency 4\n switchyard-soak --model switchyard/general --duration 48h --server-pid 1234 --max-rss-growth-mib 512\n\nThis command does not require VidaiMock, oha, or AIPerf. The optional scripts/soak_rehearsal.py command uses an embedded VidaiMock helper and warns before it starts when oha or AIPerf is missing.", version )] pub struct Args { - /// Base URL of the Switchyard server. + /// HTTP base URL of the Switchyard server under test. #[arg(long, default_value = "http://127.0.0.1:4000")] base_url: String, - /// Model id from GET /v1/models; defaults to its first model. + /// Exact route id advertised by GET /v1/models. #[arg(long)] - model: Option, + model: String, - /// Run time with an s, m, or h suffix. + /// Time to keep sending load; use an s, m, or h suffix. #[arg(long, value_parser = stats::parse_duration, default_value = "48h")] duration: f64, - /// Number of closed-loop inference workers. + /// Requests kept in flight; each worker sends its next request after the last one ends. #[arg(long, default_value_t = 16)] concurrency: usize, - /// Public APIs to exercise; defaults to all three. - #[arg(long, value_enum, num_args = 1..)] - endpoints: Vec, - - /// Fraction of inference requests that use streaming. - #[arg(long, default_value_t = 0.5)] - stream_ratio: f64, - - /// Maximum output tokens requested from the backend. + /// Output-token limit sent with every inference request. #[arg(long, default_value_t = 32)] max_output_tokens: u32, - /// Repeated-prefix payload size for each prompt. + /// Bytes of repeated prefix added to each prompt to exercise request memory and caching. #[arg(long, default_value_t = 1024)] prompt_bytes: usize, - /// Timeout in seconds for one inference request. + /// Seconds allowed to connect or wait between response bytes; an active stream may run longer. #[arg(long, default_value_t = 120.0)] request_timeout: f64, - /// Seconds between health, metrics, and result samples. + /// Seconds between health, metrics, process, and result samples. #[arg(long, default_value_t = 60.0)] report_interval: f64, - /// Seconds between invalid-request recovery checks; zero disables them. + /// Seconds between malformed-request checks; 0 turns this check off. #[arg(long, default_value_t = 300.0)] invalid_canary_interval: f64, - /// Largest allowed inference error fraction. + /// Largest passing request-error fraction, from 0 (none) through 1 (all). #[arg(long, default_value_t = 0.0)] max_error_rate: f64, - /// Local Switchyard PID to sample for RSS and CPU. + /// PID of a local switchyard-server process whose RSS and CPU should be sampled. #[arg(long)] server_pid: Option, - /// Largest allowed first-to-last RSS increase in MiB. + /// Largest passing first-to-last RSS increase in MiB; requires --server-pid. #[arg(long, requires = "server_pid")] max_rss_growth_mib: Option, - /// Environment variable holding a bearer token for the Switchyard endpoint. + /// Name of an environment variable that holds the endpoint's bearer token. #[arg(long)] api_key_env: Option, - /// New directory for the run result files. + /// New directory to create for config, interval, error, and summary files. #[arg(long)] results_dir: Option, } impl Args { - /// Reject inputs clap's types cannot, matching the Python runner's checks. + /// Reject invalid numeric combinations after clap parses their types. pub fn validate(&self) -> Result<(), String> { if self.concurrency == 0 { return Err("--concurrency must be greater than zero".to_string()); } - let mut seen = std::collections::HashSet::new(); - if !self - .endpoints - .iter() - .all(|endpoint| seen.insert(endpoint.as_str())) - { - return Err("--endpoints must not repeat a value".to_string()); - } - if !(0.0..=1.0).contains(&self.stream_ratio) { - return Err("--stream-ratio must be between 0 and 1".to_string()); - } if self.max_output_tokens == 0 || self.prompt_bytes == 0 { return Err( "--max-output-tokens and --prompt-bytes must be greater than zero".to_string(), ); } - if self.request_timeout <= 0.0 || self.report_interval <= 0.0 { + if !self.request_timeout.is_finite() + || !self.report_interval.is_finite() + || self.request_timeout <= 0.0 + || self.report_interval <= 0.0 + { return Err( "--request-timeout and --report-interval must be greater than zero".to_string(), ); } - if self.invalid_canary_interval < 0.0 { + if !self.invalid_canary_interval.is_finite() || self.invalid_canary_interval < 0.0 { return Err("--invalid-canary-interval must be zero or greater".to_string()); } if !(0.0..=1.0).contains(&self.max_error_rate) { @@ -144,46 +123,56 @@ impl Args { if self.server_pid == Some(0) { return Err("--server-pid must be greater than zero".to_string()); } - if self.max_rss_growth_mib.is_some_and(|growth| growth < 0.0) { + if self + .max_rss_growth_mib + .is_some_and(|growth| !growth.is_finite() || growth < 0.0) + { return Err("--max-rss-growth-mib must be zero or greater".to_string()); } Ok(()) } +} - fn endpoints(&self) -> Vec { - if self.endpoints.is_empty() { - Endpoint::ALL.to_vec() - } else { - self.endpoints.clone() - } - } +#[derive(Clone)] +struct RunContext { + client: Client, + base_url: String, + stop: Arc, + stats: Arc>, + writer: Arc>, +} + +struct Workload { + model: String, + prompts: Vec, + max_output_tokens: u32, } /// A one-shot stop signal that many tasks can wait on and any task can raise. -pub struct Stop { +struct Stop { flag: AtomicBool, notify: Notify, } impl Stop { - pub fn new() -> Self { + fn new() -> Self { Self { flag: AtomicBool::new(false), notify: Notify::new(), } } - pub fn set(&self) { + fn set(&self) { self.flag.store(true, Ordering::SeqCst); self.notify.notify_waiters(); } - pub fn is_set(&self) -> bool { + fn is_set(&self) -> bool { self.flag.load(Ordering::SeqCst) } /// Resolve once the signal is raised, now or later. - pub async fn wait(&self) { + async fn wait(&self) { loop { if self.is_set() { return; @@ -200,48 +189,35 @@ impl Stop { } } -impl Default for Stop { - fn default() -> Self { - Self::new() - } -} - /// Send closed-loop traffic until the run stops. -#[allow(clippy::too_many_arguments)] async fn worker( - client: Client, - base_url: String, + context: RunContext, + workload: Arc, worker_id: usize, - model: String, - endpoints: Vec, - prompt_pool: Vec, - max_output_tokens: u32, - stream_ratio: f64, - stop: Arc, request_numbers: Arc, - stats: Arc>, - writer: Arc>, ) -> Result<(), String> { - let mut rng = StdRng::seed_from_u64(10_000 + worker_id as u64); - while !stop.is_set() { + while !context.stop.is_set() { let request_number = request_numbers.fetch_add(1, Ordering::Relaxed) as usize; - let endpoint = endpoints[request_number % endpoints.len()]; - let stream = rng.random::() < stream_ratio; + let endpoint = Endpoint::ALL[request_number % Endpoint::ALL.len()]; + let stream = (request_number / Endpoint::ALL.len()).is_multiple_of(2); let body = request_body( endpoint, - &model, - &prompt_pool[request_number % prompt_pool.len()], - max_output_tokens, + &workload.model, + &workload.prompts[request_number % workload.prompts.len()], + workload.max_output_tokens, stream, ); let started = Instant::now(); - let (error_kind, detail) = send_request(&client, &base_url, endpoint, &body).await; + let result = send_request(&context.client, &context.base_url, endpoint, &body).await; let latency_ms = started.elapsed().as_secs_f64() * 1000.0; - stats - .lock() - .record(endpoint.as_str(), latency_ms, error_kind.as_deref()); - if let Some(kind) = &error_kind { - writer + context.stats.lock().record( + endpoint.as_str(), + latency_ms, + result.as_ref().err().map(|error| error.kind.as_str()), + ); + if let Err(error) = result { + context + .writer .lock() .write_error(&json!({ "timestamp_utc": now_utc_string(), @@ -249,8 +225,8 @@ async fn worker( "endpoint": endpoint.as_str(), "stream": stream, "latency_ms": round3(latency_ms), - "error": kind, - "detail": detail, + "error": error.kind, + "detail": error.detail, })) .map_err(|error| error.to_string())?; } @@ -258,10 +234,7 @@ async fn worker( Ok(()) } -/// Run a background task and raise the stop signal unless it returns `Ok`, so the run ends -/// fail-closed. The drop guard fires on an `Err` return and on a panic unwinding through the -/// await, matching the Python runner, which stopped the run on any task exception; a task that -/// returns `Ok` (a worker/reporter after stop, or a disabled canary) leaves the guard disarmed. +/// Stop the run when a background task fails or panics. async fn guard_stop( stop: Arc, task: impl Future>, @@ -280,7 +253,7 @@ async fn guard_stop( result } -/// Fold one joined task's outcome into the failure reasons; cancelled tasks are expected. +/// Add one joined task's error to the failure reasons; cancelled tasks are expected. fn collect_failure( name: &str, result: Result, tokio::task::JoinError>, @@ -295,7 +268,7 @@ fn collect_failure( } /// Raise *stop* on SIGINT or SIGTERM so an operator can end the run cleanly. -fn spawn_signal_listener(stop: Arc) { +fn spawn_signal_listener(stop: Arc) -> tokio::task::JoinHandle<()> { tokio::spawn(async move { #[cfg(unix)] { @@ -329,18 +302,18 @@ fn spawn_signal_listener(stop: Arc) { let _ = tokio::signal::ctrl_c().await; stop.set(); } - }); + }) } -/// Record every non-secret input, plus the resolved model and the seconds duration. +/// Record every non-secret input with the normalized duration and fixed request variants. fn write_config(results_dir: &Path, args: &Args, model: &str) -> Result<(), String> { let config = json!({ "base_url": args.base_url, "model": model, "duration_seconds": args.duration, "concurrency": args.concurrency, - "endpoints": args.endpoints().iter().map(|endpoint| endpoint.as_str()).collect::>(), - "stream_ratio": args.stream_ratio, + "endpoints": Endpoint::ALL.map(Endpoint::as_str), + "streaming": [true, false], "max_output_tokens": args.max_output_tokens, "prompt_bytes": args.prompt_bytes, "request_timeout": args.request_timeout, @@ -358,6 +331,8 @@ fn write_config(results_dir: &Path, args: &Args, model: &str) -> Result<(), Stri /// Run the configured soak test and return a process exit code (0 pass, 1 fail). pub async fn run(args: Args) -> Result { + args.validate()?; + let token = match &args.api_key_env { Some(var) => { let token = std::env::var(var).ok().filter(|value| !value.is_empty()); @@ -387,7 +362,7 @@ pub async fn run(args: Args) -> Result { let client = builder.build().map_err(|error| error.to_string())?; let base_url = args.base_url.trim_end_matches('/').to_string(); - let model = preflight(&client, &base_url, args.model.as_deref()).await?; + preflight(&client, &base_url, &args.model).await?; let results_dir = args .results_dir .clone() @@ -395,14 +370,14 @@ pub async fn run(args: Args) -> Result { let writer = Arc::new(Mutex::new( ResultsWriter::new(&results_dir).map_err(|error| error.to_string())?, )); - write_config(&results_dir, &args, &model)?; + write_config(&results_dir, &args, &args.model)?; - let endpoints = args.endpoints(); println!( - "Soak started: model={model} duration={}s concurrency={} endpoints={} results={}", + "Soak started: model={} duration={}s concurrency={} endpoints={} results={}", + args.model, args.duration, args.concurrency, - endpoints + Endpoint::ALL .iter() .map(|endpoint| endpoint.as_str()) .collect::>() @@ -415,8 +390,15 @@ pub async fn run(args: Args) -> Result { let stop = Arc::new(Stop::new()); let workers_done = Arc::new(Stop::new()); let request_numbers = Arc::new(AtomicU64::new(0)); + let context = RunContext { + client, + base_url, + stop: stop.clone(), + stats: stats.clone(), + writer: writer.clone(), + }; - spawn_signal_listener(stop.clone()); + let signal_listener = spawn_signal_listener(stop.clone()); let deadline = { let stop = stop.clone(); @@ -429,24 +411,20 @@ pub async fn run(args: Args) -> Result { }) }; - let prompt_pool = build_prompt_pool(args.prompt_bytes); + let workload = Arc::new(Workload { + model: args.model.clone(), + prompts: build_prompt_pool(args.prompt_bytes), + max_output_tokens: args.max_output_tokens, + }); let mut worker_handles = Vec::new(); for worker_id in 0..args.concurrency { let handle = tokio::spawn(guard_stop( stop.clone(), worker( - client.clone(), - base_url.clone(), + context.clone(), + workload.clone(), worker_id, - model.clone(), - endpoints.clone(), - prompt_pool.clone(), - args.max_output_tokens, - args.stream_ratio, - stop.clone(), request_numbers.clone(), - stats.clone(), - writer.clone(), ), )); worker_handles.push((format!("worker-{worker_id}"), handle)); @@ -454,33 +432,22 @@ pub async fn run(args: Args) -> Result { let reporter_handle = tokio::spawn(guard_stop( stop.clone(), reporter( - client.clone(), - base_url.clone(), + context.clone(), started, Duration::from_secs_f64(args.report_interval), args.duration, args.server_pid, - stop.clone(), workers_done.clone(), - stats.clone(), - writer.clone(), ), )); let canary_handle = tokio::spawn(guard_stop( stop.clone(), - invalid_request_canary( - client.clone(), - base_url.clone(), - args.invalid_canary_interval, - model.clone(), - stop.clone(), - stats.clone(), - writer.clone(), - ), + invalid_request_canary(context, args.invalid_canary_interval, workload), )); stop.wait().await; deadline.abort(); + signal_listener.abort(); // A crashed worker/reporter/canary must not discard the run: record it as a failure reason // so the summary is still written and the run fails closed. @@ -497,10 +464,7 @@ pub async fn run(args: Args) -> Result { ); let elapsed = started.elapsed().as_secs_f64(); - let (error_records, dropped_error_records) = { - let writer = writer.lock(); - (writer.error_records, writer.dropped_error_records) - }; + let (error_records, dropped_error_records) = { writer.lock().error_counts() }; let summary = build_summary( &stats.lock(), elapsed, @@ -510,10 +474,8 @@ pub async fn run(args: Args) -> Result { dropped_error_records, &task_failures, ); - let latency_block = latency_report(stats.lock().latency_samples()); let summary_path = results_dir.join("summary.json"); - let summary_body = - serde_json::to_string_pretty(&summary.json).map_err(|error| error.to_string())?; + let summary_body = serde_json::to_string_pretty(&summary).map_err(|error| error.to_string())?; fs::write(&summary_path, format!("{summary_body}\n")).map_err(|error| error.to_string())?; let label = if summary.passed { "PASS" } else { "FAIL" }; @@ -527,12 +489,9 @@ pub async fn run(args: Args) -> Result { .unwrap_or_default(), summary_path.display(), ); - for reason in &summary.reasons { + for reason in &summary.failure_reasons { println!("- {reason}"); } - if !latency_block.is_empty() { - print!("{latency_block}"); - } Ok(if summary.passed { 0 } else { 1 }) } diff --git a/crates/switchyard-soak/src/report.rs b/crates/switchyard-soak/src/report.rs index bfcfe74b7..b941e4ea5 100644 --- a/crates/switchyard-soak/src/report.rs +++ b/crates/switchyard-soak/src/report.rs @@ -5,19 +5,14 @@ use std::fs::{self, File}; use std::io::{self, Write}; -use std::path::{Path, PathBuf}; -use std::sync::Arc; +use std::path::Path; use std::time::{Duration, Instant}; -use parking_lot::Mutex; -use reqwest::Client; use serde_json::{Value, json}; -use crate::Stop; use crate::client::read_server_state; -use crate::stats::{ - RunStats, SERVER_ERRORS_METRIC, SERVER_REQUESTS_METRIC, now_utc_string, percentile, round3, -}; +use crate::stats::{latency_stats, now_utc_string, round3}; +use crate::{RunContext, Stop, Workload}; /// Cap on individually recorded failures so a bad run cannot fill the disk. const MAX_ERROR_RECORDS: u64 = 10_000; @@ -42,11 +37,10 @@ const INTERVAL_FIELDS: [&str; 15] = [ /// Write interval rows and bounded error details as the run proceeds. pub struct ResultsWriter { - pub results_dir: PathBuf, interval_file: File, error_file: File, - pub error_records: u64, - pub dropped_error_records: u64, + error_records: u64, + dropped_error_records: u64, } impl ResultsWriter { @@ -62,7 +56,6 @@ impl ResultsWriter { writeln!(interval_file, "{}", INTERVAL_FIELDS.join(","))?; let error_file = File::create(results_dir.join("errors.jsonl"))?; Ok(Self { - results_dir: results_dir.to_path_buf(), interval_file, error_file, error_records: 0, @@ -82,20 +75,14 @@ impl ResultsWriter { self.dropped_error_records += 1; return Ok(()); } - // serde_json's map is ordered, so keys serialize sorted, matching the run config file. writeln!(self.error_file, "{record}")?; self.error_file.flush()?; self.error_records += 1; Ok(()) } - /// Overwrite status.json with the current run snapshot, written atomically (write-then-rename) - /// so a remote monitor can read a complete file at any moment without racing the writer. - pub fn write_status(&self, snapshot: &Value) -> io::Result<()> { - let body = serde_json::to_string_pretty(snapshot).map_err(io::Error::other)?; - let tmp = self.results_dir.join("status.json.tmp"); - fs::write(&tmp, format!("{body}\n"))?; - fs::rename(tmp, self.results_dir.join("status.json")) + pub fn error_counts(&self) -> (u64, u64) { + (self.error_records, self.dropped_error_records) } } @@ -120,59 +107,53 @@ pub async fn process_sample(pid: Option) -> (Option, Option) { // ps missing or fork failed: record a process-check miss, don't crash the reporter. Err(_) => return (None, None), }; - let fields: Vec = String::from_utf8_lossy(&output.stdout) - .split_whitespace() - .map(str::to_string) - .collect(); - if !output.status.success() || fields.len() != 2 { + if !output.status.success() { return (None, None); } - match (fields[0].parse::(), fields[1].parse::()) { + let stdout = String::from_utf8_lossy(&output.stdout); + let mut fields = stdout.split_whitespace(); + let (Some(rss), Some(cpu), None) = (fields.next(), fields.next(), fields.next()) else { + return (None, None); + }; + match (rss.parse::(), cpu.parse::()) { (Ok(rss_kib), Ok(cpu_percent)) => (Some(rss_kib / 1024.0), Some(cpu_percent)), _ => (None, None), } } /// Write one liveness, metrics, resource, and latency row per interval. -#[allow(clippy::too_many_arguments)] pub async fn reporter( - client: Client, - base_url: String, + context: RunContext, started: Instant, interval: Duration, target_seconds: f64, server_pid: Option, - stop: Arc, - workers_done: Arc, - stats: Arc>, - writer: Arc>, + workers_done: std::sync::Arc, ) -> Result<(), String> { let mut previous_report = started; loop { tokio::select! { _ = tokio::time::sleep(interval) => {} - _ = stop.wait() => {} + _ = context.stop.wait() => {} } // Once stopping, wait for the workers to drain so the final row counts their last requests. - if stop.is_set() { + if context.stop.is_set() { workers_done.wait().await; } let now = Instant::now(); - let interval_stats = stats.lock().take_interval(); - let (healthy, metrics) = read_server_state(&client, &base_url).await; + let mut interval_stats = context.stats.lock().take_interval(); + let server = read_server_state(&context.client, &context.base_url).await; let (rss_mib, cpu_percent) = process_sample(server_pid).await; - let server_requests = metrics.get(SERVER_REQUESTS_METRIC).copied(); - let server_errors = metrics.get(SERVER_ERRORS_METRIC).copied(); - let (total_successes, total_failures, server_restarts, completed_duration) = { - let mut state = stats.lock(); + let (total_successes, total_failures) = { + let mut state = context.stats.lock(); state.health_checks += 1; state.metrics_checks += 1; - if !healthy { + if !server.healthy { state.health_failures += 1; } - if server_requests.is_none() || server_errors.is_none() { + if server.requests.is_none() || server.errors.is_none() { state.metrics_failures += 1; } if server_pid.is_some() { @@ -185,36 +166,24 @@ pub async fn reporter( state.rss_samples.push(rss); } if let (Some(current), Some(previous)) = - (server_requests, state.previous_server_requests) + (server.requests, state.previous_server_requests) && current < previous { state.server_restarts += 1; } - if let Some(current) = server_requests { + if let Some(current) = server.requests { state.previous_server_requests = Some(current); } - ( - state.total_successes, - state.total_failures, - state.server_restarts, - state.completed_duration, - ) + (state.total_successes, state.total_failures) }; let elapsed_interval = (now - previous_report).as_secs_f64().max(0.001); let requests = interval_stats.successes + interval_stats.failures; - let latency_max = interval_stats - .latencies_ms - .iter() - .copied() - .fold(None, |acc: Option, value| { - Some(acc.map_or(value, |m: f64| m.max(value))) - }); + let latency = latency_stats(&mut interval_stats.latencies_ms); let requests_per_second = round3(requests as f64 / elapsed_interval); let timestamp = now_utc_string(); let elapsed_seconds = round3((now - started).as_secs_f64()); - let health_label = if healthy { "ok" } else { "failed" }; - let latency_p95 = percentile(&interval_stats.latencies_ms, 0.95).map(round3); + let health_label = if server.healthy { "ok" } else { "failed" }; let cells = vec![ timestamp.clone(), @@ -223,13 +192,13 @@ pub async fn reporter( interval_stats.successes.to_string(), interval_stats.failures.to_string(), requests_per_second.to_string(), - cell(percentile(&interval_stats.latencies_ms, 0.50)), - cell(latency_p95), - cell(percentile(&interval_stats.latencies_ms, 0.99)), - cell(latency_max), + cell(latency.p50_ms), + cell(latency.p95_ms), + cell(latency.p99_ms), + cell(latency.max_ms), health_label.to_string(), - cell(server_requests), - cell(server_errors), + cell(server.requests), + cell(server.errors), cell(rss_mib), cell(cpu_percent), ]; @@ -241,46 +210,22 @@ pub async fn reporter( } else { 0.0 }; - let progress = if target_seconds > 0.0 { - (elapsed_seconds / target_seconds).min(1.0) - } else { - 0.0 - }; + let progress = (elapsed_seconds / target_seconds).min(1.0); let status = if requests == 0 { "stalled" - } else if !healthy || interval_stats.failures > 0 { + } else if !server.healthy || interval_stats.failures > 0 { "degraded" } else { "ok" }; - let p95_text = latency_p95.map(|v| v.to_string()).unwrap_or_default(); + let p95_text = latency.p95_ms.map(|v| v.to_string()).unwrap_or_default(); let rss_text = rss_mib.map(|v| round3(v).to_string()).unwrap_or_default(); - let snapshot = json!({ - "timestamp_utc": timestamp, - "elapsed_seconds": elapsed_seconds, - "target_seconds": round3(target_seconds), - "progress": round3(progress), - "requests": cumulative_requests, - "successes": total_successes, - "failures": total_failures, - "error_rate": cumulative_error_rate, - "interval_requests": requests, - "requests_per_second": requests_per_second, - "latency_p95_ms": latency_p95, - "health": health_label, - "rss_mib": rss_mib.map(round3), - "detected_server_restarts": server_restarts, - "status": status, - "completed_duration": completed_duration, - }); - { - let mut w = writer.lock(); - w.write_interval(&cells) - .map_err(|error| error.to_string())?; - w.write_status(&snapshot) - .map_err(|error| error.to_string())?; - } + context + .writer + .lock() + .write_interval(&cells) + .map_err(|error| error.to_string())?; println!( "[{timestamp}] progress={elapsed_seconds:.0}s/{target_seconds:.0}s({:.0}%) \ @@ -293,7 +238,7 @@ pub async fn reporter( ); previous_report = now; - if stop.is_set() { + if context.stop.is_set() { return Ok(()); } } @@ -301,13 +246,9 @@ pub async fn reporter( /// Confirm invalid input returns 400 and the server stays live, on a fixed interval. pub async fn invalid_request_canary( - client: Client, - base_url: String, + context: RunContext, interval: f64, - model: String, - stop: Arc, - stats: Arc>, - writer: Arc>, + workload: std::sync::Arc, ) -> Result<(), String> { if interval <= 0.0 { return Ok(()); @@ -316,20 +257,25 @@ pub async fn invalid_request_canary( loop { tokio::select! { _ = tokio::time::sleep(interval) => {} - _ = stop.wait() => {} + _ = context.stop.wait() => {} } - if stop.is_set() { + if context.stop.is_set() { return Ok(()); } - stats.lock().canaries += 1; + context.stats.lock().canaries += 1; let probe = async { - let invalid = client - .post(format!("{base_url}/v1/chat/completions")) - .json(&json!({"model": model, "messages": []})) + let invalid = context + .client + .post(format!("{}/v1/chat/completions", context.base_url)) + .json(&json!({"model": workload.model, "messages": []})) + .send() + .await?; + let health = context + .client + .get(format!("{}/health", context.base_url)) .send() .await?; - let health = client.get(format!("{base_url}/health")).send().await?; Ok::<(u16, u16), reqwest::Error>((invalid.status().as_u16(), health.status().as_u16())) } .await; @@ -341,8 +287,9 @@ pub async fn invalid_request_canary( Err(error) => (false, error.to_string()), }; if !passed { - stats.lock().canary_failures += 1; - writer + context.stats.lock().canary_failures += 1; + context + .writer .lock() .write_error(&json!({ "timestamp_utc": now_utc_string(), @@ -353,3 +300,28 @@ pub async fn invalid_request_canary( } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn error_details_stop_growing_at_the_file_limit() -> io::Result<()> { + let parent = tempfile::tempdir()?; + let results_dir = parent.path().join("results"); + let mut writer = ResultsWriter::new(&results_dir)?; + + for _ in 0..=MAX_ERROR_RECORDS { + writer.write_error(&serde_json::json!({"error": "upstream"}))?; + } + + assert_eq!(writer.error_counts(), (MAX_ERROR_RECORDS, 1)); + assert_eq!( + fs::read_to_string(results_dir.join("errors.jsonl"))? + .lines() + .count(), + 10_000 + ); + Ok(()) + } +} diff --git a/crates/switchyard-soak/src/stats.rs b/crates/switchyard-soak/src/stats.rs index 57ed29f73..e5b38412d 100644 --- a/crates/switchyard-soak/src/stats.rs +++ b/crates/switchyard-soak/src/stats.rs @@ -4,19 +4,15 @@ //! Bounded in-memory run state, latency percentiles, and the final pass/fail summary. use std::collections::BTreeMap; -use std::time::{SystemTime, UNIX_EPOCH}; +use std::time::SystemTime; use rand::rngs::StdRng; use rand::{RngExt, SeedableRng}; -use serde_json::{Value, json}; +use serde::Serialize; /// Cap on retained latency samples; a long run stays within this bound by reservoir sampling. const RESERVOIR_SIZE: usize = 100_000; -/// Cumulative counter names Switchyard exposes on `/metrics`. -pub const SERVER_REQUESTS_METRIC: &str = "switchyard_total_requests"; -pub const SERVER_ERRORS_METRIC: &str = "switchyard_total_errors"; - /// Round to three decimals so result files stay readable. pub fn round3(value: f64) -> f64 { (value * 1000.0).round() / 1000.0 @@ -33,106 +29,35 @@ pub fn parse_duration(value: &str) -> Result { 'h' => 3600.0, _ => return Err(bad()), }; - // Accept only `\d+(\.\d+)?` before the unit: at least one digit, at most one dot with - // digits on both sides. This rejects "10", "5x", "-3s", "1.", and ".5". let number = &text[..text.len() - unit.len_utf8()]; - let parts: Vec<&str> = number.split('.').collect(); - let well_formed = matches!(parts.len(), 1 | 2) - && parts - .iter() - .all(|part| !part.is_empty() && part.bytes().all(|b| b.is_ascii_digit())); - if !well_formed { - return Err(bad()); - } let seconds = number.parse::().map_err(|_| bad())? * multiplier; - if seconds <= 0.0 { + if !seconds.is_finite() || seconds <= 0.0 { return Err("duration must be greater than zero".to_string()); } Ok(seconds) } -/// Return the nearest-rank percentile for *values*, or `None` when empty. -pub fn percentile(values: &[f64], quantile: f64) -> Option { - if values.is_empty() { - return None; - } - let mut ordered = values.to_vec(); - ordered.sort_by(|a, b| a.total_cmp(b)); - // round_ties_even matches Python's round(), which breaks exact halves to even. - let index = (((ordered.len() - 1) as f64) * quantile).round_ties_even() as usize; - Some(ordered[index]) +#[derive(Default)] +pub struct LatencyStats { + pub p50_ms: Option, + pub p95_ms: Option, + pub p99_ms: Option, + pub max_ms: Option, } -/// One equal-width latency bucket for the end-of-run histogram. -struct Bucket { - low: f64, - high: f64, - count: usize, +fn percentile_sorted(values: &[f64], quantile: f64) -> Option { + let index = (((values.len().checked_sub(1)?) as f64) * quantile).round_ties_even() as usize; + Some(round3(values[index])) } -/// Split *values* into *bucket_count* equal-width buckets between the min and max sample. -/// The maximum sample falls in the last bucket rather than spilling past it. -fn histogram(values: &[f64], bucket_count: usize) -> Vec { - let min = values.iter().copied().fold(f64::INFINITY, f64::min); - let max = values.iter().copied().fold(f64::NEG_INFINITY, f64::max); - let width = ((max - min) / bucket_count as f64).max(f64::MIN_POSITIVE); - let mut counts = vec![0usize; bucket_count]; - for &value in values { - let index = (((value - min) / width) as usize).min(bucket_count - 1); - counts[index] += 1; +pub fn latency_stats(values: &mut [f64]) -> LatencyStats { + values.sort_by(|a, b| a.total_cmp(b)); + LatencyStats { + p50_ms: percentile_sorted(values, 0.50), + p95_ms: percentile_sorted(values, 0.95), + p99_ms: percentile_sorted(values, 0.99), + max_ms: values.last().copied().map(round3), } - counts - .into_iter() - .enumerate() - .map(|(index, count)| Bucket { - low: min + index as f64 * width, - high: min + (index + 1) as f64 * width, - count, - }) - .collect() -} - -/// Render an oha-style latency percentile table and ASCII histogram, or empty when no samples. -/// Built from the bounded reservoir, so it reflects the whole run within the reservoir cap. -pub fn latency_report(reservoir: &[f64]) -> String { - if reservoir.is_empty() { - return String::new(); - } - let mut out = String::from("Latency distribution (ms):\n"); - for (label, quantile) in [ - ("p50", 0.50), - ("p75", 0.75), - ("p90", 0.90), - ("p95", 0.95), - ("p99", 0.99), - ("p99.9", 0.999), - ] { - if let Some(value) = percentile(reservoir, quantile) { - out.push_str(&format!(" {label:<6}{:>12.3}\n", round3(value))); - } - } - out.push_str("Latency histogram (ms):\n"); - let buckets = histogram(reservoir, 10); - let widest = buckets - .iter() - .map(|bucket| bucket.count) - .max() - .unwrap_or(1) - .max(1); - for bucket in &buckets { - let bar = "\u{25a0}".repeat(bucket.count * 40 / widest); - out.push_str(&format!( - " {:>9.1} - {:>9.1} [{:>7}] {bar}\n", - round3(bucket.low), - round3(bucket.high), - bucket.count - )); - } - out -} - -fn rounded_percentile(values: &[f64], quantile: f64) -> Option { - percentile(values, quantile).map(round3) } /// Return four stable prompts with reusable prefixes of *prompt_bytes* filler each. @@ -151,51 +76,15 @@ pub fn build_prompt_pool(prompt_bytes: usize) -> Vec { /// ISO-8601 UTC timestamp at second precision, e.g. `2026-07-30T18:04:00Z`. pub fn now_utc_string() -> String { - format_utc(SystemTime::now()) + humantime::format_rfc3339_seconds(SystemTime::now()).to_string() } /// Compact UTC stamp for a results directory, e.g. `20260730T180400Z`. pub fn utc_dir_stamp() -> String { - let secs = unix_seconds(SystemTime::now()); - let (year, month, day, hour, minute, second) = civil_from_unix(secs); - format!("{year:04}{month:02}{day:02}T{hour:02}{minute:02}{second:02}Z") -} - -fn format_utc(time: SystemTime) -> String { - let (year, month, day, hour, minute, second) = civil_from_unix(unix_seconds(time)); - format!("{year:04}-{month:02}-{day:02}T{hour:02}:{minute:02}:{second:02}Z") -} - -fn unix_seconds(time: SystemTime) -> i64 { - // Before the epoch never happens for a live run; clamp to 0 rather than panic. - time.duration_since(UNIX_EPOCH) - .map(|d| d.as_secs() as i64) - .unwrap_or(0) -} - -/// Split seconds-since-epoch into UTC (year, month, day, hour, minute, second). -fn civil_from_unix(secs: i64) -> (i64, u32, u32, u32, u32, u32) { - let days = secs.div_euclid(86_400); - let rem = secs.rem_euclid(86_400); - let (hour, minute, second) = (rem / 3600, rem % 3600 / 60, rem % 60); - // Howard Hinnant's days-to-civil algorithm. - let z = days + 719_468; - let era = if z >= 0 { z } else { z - 146_096 } / 146_097; - let doe = z - era * 146_097; - let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365; - let year = yoe + era * 400; - let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); - let mp = (5 * doy + 2) / 153; - let day = (doy - (153 * mp + 2) / 5 + 1) as u32; - let month = if mp < 10 { mp + 3 } else { mp - 9 } as u32; - ( - year + i64::from(month <= 2), - month, - day, - hour as u32, - minute as u32, - second as u32, - ) + now_utc_string() + .chars() + .filter(|character| !matches!(character, '-' | ':')) + .collect() } /// Request results collected since the previous report. @@ -295,21 +184,41 @@ impl RunStats { pub fn take_interval(&mut self) -> IntervalStats { std::mem::take(&mut self.interval) } - - /// The bounded reservoir of latency samples, for the end-of-run distribution report. - pub fn latency_samples(&self) -> &[f64] { - &self.latency_reservoir - } } -/// Final result of a run: the exit-relevant fields plus the JSON written to `summary.json`. +/// Final result written to `summary.json`. +#[derive(Serialize)] pub struct Summary { pub passed: bool, - pub reasons: Vec, + pub failure_reasons: Vec, + pub completed_duration: bool, + pub elapsed_seconds: f64, pub requests: u64, + pub successes: u64, + pub failures: u64, pub error_rate: f64, + pub requests_per_second: f64, + pub endpoint_successes: BTreeMap, + pub endpoint_failures: BTreeMap, + pub error_kinds: BTreeMap, + pub latency_p50_ms: Option, pub latency_p95_ms: Option, - pub json: Value, + pub latency_p99_ms: Option, + pub health_checks: u64, + pub health_failures: u64, + pub metrics_checks: u64, + pub metrics_failures: u64, + pub process_checks: u64, + pub process_failures: u64, + pub invalid_request_canaries: u64, + pub invalid_request_canary_failures: u64, + pub detected_server_restarts: u64, + pub rss_first_mib: Option, + pub rss_last_mib: Option, + pub rss_max_mib: Option, + pub rss_growth_mib: Option, + pub error_records: u64, + pub dropped_error_records: u64, } /// Build the final result and its release-gate reasons. @@ -397,46 +306,40 @@ pub fn build_summary( 0.0 }; - let json = json!({ - "passed": reasons.is_empty(), - "failure_reasons": reasons, - "completed_duration": stats.completed_duration, - "elapsed_seconds": round3(elapsed_seconds), - "requests": total, - "successes": stats.total_successes, - "failures": stats.total_failures, - "error_rate": error_rate, - "requests_per_second": requests_per_second, - "endpoint_successes": stats.endpoint_successes, - "endpoint_failures": stats.endpoint_failures, - "error_kinds": stats.error_kinds, - "latency_p50_ms": rounded_percentile(&stats.latency_reservoir, 0.50), - "latency_p95_ms": rounded_percentile(&stats.latency_reservoir, 0.95), - "latency_p99_ms": rounded_percentile(&stats.latency_reservoir, 0.99), - "health_checks": stats.health_checks, - "health_failures": stats.health_failures, - "metrics_checks": stats.metrics_checks, - "metrics_failures": stats.metrics_failures, - "process_checks": stats.process_checks, - "process_failures": stats.process_failures, - "invalid_request_canaries": stats.canaries, - "invalid_request_canary_failures": stats.canary_failures, - "detected_server_restarts": stats.server_restarts, - "rss_first_mib": rss_first, - "rss_last_mib": rss_last, - "rss_max_mib": rss_max, - "rss_growth_mib": rss_growth, - "error_records": error_records, - "dropped_error_records": dropped_error_records, - }); + let mut latency_samples = stats.latency_reservoir.clone(); + let latency = latency_stats(&mut latency_samples); Summary { passed: reasons.is_empty(), - reasons, + failure_reasons: reasons, + completed_duration: stats.completed_duration, + elapsed_seconds: round3(elapsed_seconds), requests: total, + successes: stats.total_successes, + failures: stats.total_failures, error_rate, - latency_p95_ms: rounded_percentile(&stats.latency_reservoir, 0.95), - json, + requests_per_second, + endpoint_successes: stats.endpoint_successes.clone(), + endpoint_failures: stats.endpoint_failures.clone(), + error_kinds: stats.error_kinds.clone(), + latency_p50_ms: latency.p50_ms, + latency_p95_ms: latency.p95_ms, + latency_p99_ms: latency.p99_ms, + health_checks: stats.health_checks, + health_failures: stats.health_failures, + metrics_checks: stats.metrics_checks, + metrics_failures: stats.metrics_failures, + process_checks: stats.process_checks, + process_failures: stats.process_failures, + invalid_request_canaries: stats.canaries, + invalid_request_canary_failures: stats.canary_failures, + detected_server_restarts: stats.server_restarts, + rss_first_mib: rss_first, + rss_last_mib: rss_last, + rss_max_mib: rss_max, + rss_growth_mib: rss_growth, + error_records, + dropped_error_records, } } @@ -445,116 +348,98 @@ mod tests { use super::*; #[test] - fn parse_duration_reads_suffixes() { + fn parse_duration_accepts_units_and_rejects_invalid_values() { assert_eq!(parse_duration("30s"), Ok(30.0)); + assert_eq!(parse_duration(".5s"), Ok(0.5)); assert_eq!(parse_duration("2.5m"), Ok(150.0)); assert_eq!(parse_duration("48h"), Ok(172_800.0)); - } - - #[test] - fn parse_duration_rejects_bad_values() { - for value in ["0s", "10", "5x", "-3s", "1d", "1.", ".5"] { + for value in ["0s", "10", "5x", "-3s", "1d", "NaNs", "infs"] { assert!(parse_duration(value).is_err(), "{value} should be rejected"); } } #[test] - fn percentile_uses_nearest_rank() { - let values = [1.0, 2.0, 3.0, 4.0]; - assert_eq!(percentile(&values, 0.0), Some(1.0)); - assert_eq!(percentile(&values, 0.95), Some(4.0)); - assert_eq!(percentile(&[], 0.5), None); - // Exact half -> even rank, matching Python round(): (6-1)*0.5 = 2.5 rounds to 2, not 3. - assert_eq!( - percentile(&[10.0, 20.0, 30.0, 40.0, 50.0, 60.0], 0.5), - Some(30.0) - ); + fn latency_stats_sorts_once_and_handles_empty_input() { + let mut values = [60.0, 10.0, 50.0, 20.0, 40.0, 30.0]; + let latency = latency_stats(&mut values); + assert_eq!(latency.p50_ms, Some(30.0)); + assert_eq!(latency.p95_ms, Some(60.0)); + assert_eq!(latency.p99_ms, Some(60.0)); + assert_eq!(latency.max_ms, Some(60.0)); + + let empty = latency_stats(&mut []); + assert_eq!(empty.p50_ms, None); + assert_eq!(empty.max_ms, None); } #[test] - fn format_utc_matches_known_instants() { - let epoch = SystemTime::UNIX_EPOCH; - assert_eq!(format_utc(epoch), "1970-01-01T00:00:00Z"); - // 2023-11-14T22:13:20Z - let later = epoch + std::time::Duration::from_secs(1_700_000_000); - assert_eq!(format_utc(later), "2023-11-14T22:13:20Z"); - } - - #[test] - fn histogram_buckets_cover_all_samples() { - let values = [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0]; - let buckets = histogram(&values, 10); - assert_eq!(buckets.len(), 10); - // Every sample is counted exactly once, and the max lands in the last bucket. - assert_eq!(buckets.iter().map(|b| b.count).sum::(), values.len()); - assert_eq!(buckets[9].count, 2); // 9.0 and 10.0 (the max) share the last bucket - assert_eq!(buckets[0].low, 0.0); - - // A single repeated value does not panic and lands entirely in one bucket. - let flat = histogram(&[5.0, 5.0, 5.0], 10); - assert_eq!(flat.iter().map(|b| b.count).sum::(), 3); + fn record_tracks_interval_and_cumulative_results() { + let mut stats = RunStats::new(1); + stats.record("chat", 10.0, None); + stats.record("messages", 20.0, Some("timeout")); + + let interval = stats.take_interval(); + assert_eq!(interval.successes, 1); + assert_eq!(interval.failures, 1); + assert_eq!(stats.total_successes, 1); + assert_eq!(stats.total_failures, 1); + assert_eq!(stats.error_kinds.get("timeout"), Some(&1)); } #[test] - fn latency_report_is_empty_without_samples() { - assert_eq!(latency_report(&[]), ""); - assert!(latency_report(&[1.0, 2.0, 3.0]).contains("Latency histogram (ms):")); - } + fn latency_samples_stay_bounded_after_the_reservoir_fills() { + let mut stats = RunStats::new(1); + for sample in 0..=RESERVOIR_SIZE { + stats.record("chat", sample as f64, None); + } - #[test] - fn build_prompt_pool_sizes_filler() { - let pool = build_prompt_pool(32); - assert_eq!(pool.len(), 4); - assert!(pool[0].starts_with("Switchyard soak prefix 0. ")); - assert!(pool[0].ends_with("Reply with exactly OK. ")); + assert_eq!(stats.latency_reservoir.len(), RESERVOIR_SIZE); + assert_eq!(stats.latency_count, RESERVOIR_SIZE as u64 + 1); } #[test] - fn summary_fails_on_restart_and_error_budget() { + fn summary_reports_all_failed_gates() { let mut stats = RunStats::new(1); - stats.completed_duration = true; stats.total_successes = 998; stats.total_failures = 2; + stats.health_failures = 1; + stats.metrics_failures = 1; + stats.process_failures = 1; + stats.canary_failures = 1; stats.server_restarts = 1; - - let summary = build_summary(&stats, 10.0, 0.001, None, 0, 0, &[]); - - assert!(!summary.passed); - assert!(summary.reasons.iter().any(|r| r.contains("error rate"))); - assert!(summary.reasons.iter().any(|r| r.contains("counters reset"))); - } - - #[test] - fn summary_records_task_failures() { - let mut stats = RunStats::new(1); - stats.completed_duration = true; - stats.total_successes = 100; + stats.rss_samples = vec![100.0, 700.0]; let summary = build_summary( &stats, 10.0, - 0.0, - None, + 0.001, + Some(512.0), 0, 0, &["reporter failed: boom".to_string()], ); assert!(!summary.passed); - assert!(summary.reasons.iter().any(|r| r == "reporter failed: boom")); - } - - #[test] - fn summary_flags_rss_growth() { - let mut stats = RunStats::new(1); - stats.completed_duration = true; - stats.total_successes = 100; - stats.rss_samples = vec![100.0, 700.0]; - - let summary = build_summary(&stats, 10.0, 0.0, Some(512.0), 0, 0, &[]); - - assert!(!summary.passed); - assert_eq!(summary.json["rss_growth_mib"], json!(600.0)); - assert!(summary.reasons.iter().any(|r| r.contains("RSS grew"))); + for reason in [ + "run stopped", + "error rate", + "liveness", + "metrics", + "process", + "invalid-request", + "counters reset", + "RSS grew", + "reporter failed", + ] { + assert!( + summary + .failure_reasons + .iter() + .any(|message| message.contains(reason)), + "missing {reason:?}: {:?}", + summary.failure_reasons, + ); + } + assert_eq!(summary.rss_growth_mib, Some(600.0)); } } diff --git a/crates/switchyard-soak/tests/soak.rs b/crates/switchyard-soak/tests/soak.rs index 6c28dda9c..74fd288ed 100644 --- a/crates/switchyard-soak/tests/soak.rs +++ b/crates/switchyard-soak/tests/soak.rs @@ -1,14 +1,14 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! End-to-end tests that drive the soak runner against a hermetic axum mock of a Switchyard server. +//! End-to-end tests against a local server that implements the public Switchyard APIs. +use std::collections::HashSet; use std::error::Error; use std::sync::Arc; -use std::time::Duration; use axum::Router; -use axum::extract::Json; +use axum::extract::{Json, State}; use axum::http::{StatusCode, header}; use axum::response::{IntoResponse, Response}; use axum::routing::{get, post}; @@ -16,38 +16,47 @@ use clap::Parser; use parking_lot::Mutex; use serde_json::{Value, json}; -use switchyard_soak::client::{Endpoint, preflight, read_server_state, request_body, send_request}; -use switchyard_soak::report::{ResultsWriter, invalid_request_canary}; -use switchyard_soak::stats::RunStats; -use switchyard_soak::{Args, Stop, run}; +use switchyard_soak::{Args, run}; type TestResult = Result<(), Box>; -/// Bind an ephemeral port, serve *app*, and return its base URL. The listener is bound before the -/// server task starts, so client connections queue rather than being refused. -async fn serve(app: Router) -> Result> { +struct TestServer { + base_url: String, + task: tokio::task::JoinHandle<()>, +} + +impl Drop for TestServer { + fn drop(&mut self) { + self.task.abort(); + } +} + +async fn serve(app: Router) -> Result> { let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; let addr = listener.local_addr()?; - tokio::spawn(async move { + let task = tokio::spawn(async move { let _ = axum::serve(listener, app.into_make_service()).await; }); - Ok(format!("http://{addr}")) + Ok(TestServer { + base_url: format!("http://{addr}"), + task, + }) } -fn client() -> Result> { - Ok(reqwest::Client::builder().no_proxy().build()?) +#[derive(Clone, Default)] +struct MockState { + is_chat_broken: bool, + seen: Arc>>, } -fn sse_body(body: &'static str) -> Response { - ([(header::CONTENT_TYPE, "text/event-stream")], body).into_response() -} - -fn is_stream(body: &Value) -> bool { - body.get("stream").and_then(Value::as_bool).unwrap_or(false) +impl MockState { + fn record(&self, endpoint: &str, body: &Value) { + let stream = body.get("stream").and_then(Value::as_bool) == Some(true); + self.seen.lock().insert(format!("{endpoint}:{stream}")); + } } -/// A well-behaved Switchyard: valid shapes, SSE for streaming, HTTP 400 for empty messages. -fn healthy_switchyard() -> Router { +fn switchyard(is_chat_broken: bool) -> (Router, Arc>>) { async fn health() -> Response { Json(json!({"status": "ok"})).into_response() } @@ -57,262 +66,174 @@ fn healthy_switchyard() -> Router { async fn models() -> Response { Json(json!({"data": [{"id": "soak-route"}]})).into_response() } - async fn chat(Json(body): Json) -> Response { + async fn chat(State(state): State, Json(body): Json) -> Response { let empty = body .get("messages") .and_then(Value::as_array) - .map(|messages| messages.is_empty()) - .unwrap_or(false); + .is_some_and(Vec::is_empty); if empty { - return (StatusCode::BAD_REQUEST, "messages must not be empty").into_response(); + return if state.is_chat_broken { + Json(json!({"choices": []})).into_response() + } else { + (StatusCode::BAD_REQUEST, "messages must not be empty").into_response() + }; } - if is_stream(&body) { - sse_body("data: {}\n\ndata: [DONE]\n\n") - } else { - Json(json!({"choices": []})).into_response() + state.record("chat", &body); + if state.is_chat_broken { + return if body.get("stream") == Some(&Value::Bool(true)) { + Json(json!({"choices": []})).into_response() + } else { + Json(json!({})).into_response() + }; } + response(&body, "choices") } - async fn messages(Json(body): Json) -> Response { - if is_stream(&body) { - sse_body("data: {}\n\ndata: [DONE]\n\n") - } else { - Json(json!({"content": []})).into_response() - } + async fn messages(State(state): State, Json(body): Json) -> Response { + state.record("messages", &body); + response(&body, "content") } - async fn responses(Json(body): Json) -> Response { - if is_stream(&body) { - sse_body("data: {}\n\ndata: [DONE]\n\n") + async fn responses(State(state): State, Json(body): Json) -> Response { + state.record("responses", &body); + response(&body, "output") + } + fn response(body: &Value, field: &str) -> Response { + if body.get("stream") == Some(&Value::Bool(true)) { + ( + [(header::CONTENT_TYPE, "text/event-stream")], + "data: {}\n\n", + ) + .into_response() } else { - Json(json!({"output": []})).into_response() + let mut response = serde_json::Map::new(); + response.insert(field.to_string(), json!([])); + Json(Value::Object(response)).into_response() } } - Router::new() + + let state = MockState { + is_chat_broken, + ..MockState::default() + }; + let seen = state.seen.clone(); + let app = Router::new() .route("/health", get(health)) .route("/metrics", get(metrics)) .route("/v1/models", get(models)) .route("/v1/chat/completions", post(chat)) .route("/v1/messages", post(messages)) .route("/v1/responses", post(responses)) + .with_state(state); + (app, seen) } -#[tokio::test] -async fn short_run_passes_against_live_switchyard_routes() -> TestResult { - let base_url = serve(healthy_switchyard()).await?; - let dir = tempfile::tempdir()?; - let results_dir = dir.path().join("soak-results"); - - let args = Args::try_parse_from([ +fn args(base_url: &str, results_dir: &str) -> Result { + Args::try_parse_from([ "switchyard-soak", "--base-url", - &base_url, + base_url, "--model", "soak-route", "--duration", - "1s", + "0.8s", "--concurrency", "2", - "--stream-ratio", - "0.5", "--report-interval", - "0.2", + "0.1", "--invalid-canary-interval", - "0.2", + "0.1", "--results-dir", + results_dir, + ]) +} + +#[tokio::test] +async fn short_run_exercises_every_response_variant_and_passes() -> TestResult { + let (app, seen) = switchyard(false); + let server = serve(app).await?; + let dir = tempfile::tempdir()?; + let results_dir = dir.path().join("soak-results"); + let args = args( + &server.base_url, results_dir.to_str().ok_or("non-utf8 results dir")?, - ])?; + )?; + args.validate()?; assert_eq!(run(args).await?, 0); let summary: Value = serde_json::from_str(&std::fs::read_to_string(results_dir.join("summary.json"))?)?; assert_eq!(summary["passed"], json!(true)); - for endpoint in ["chat", "messages", "responses"] { - assert!( - summary["endpoint_successes"][endpoint] - .as_u64() - .unwrap_or(0) - > 0, - "expected successes on {endpoint}: {summary}" - ); - } assert!(summary["invalid_request_canaries"].as_u64().unwrap_or(0) > 0); + assert_eq!( + *seen.lock(), + HashSet::from([ + "chat:false".to_string(), + "chat:true".to_string(), + "messages:false".to_string(), + "messages:true".to_string(), + "responses:false".to_string(), + "responses:true".to_string(), + ]) + ); Ok(()) } #[tokio::test] -async fn preflight_selects_default_model() -> TestResult { - async fn models() -> Response { - Json(json!({"data": [{"id": "route-a"}, {"id": "route-b"}], "default_model": "route-b"})) - .into_response() - } - let app = Router::new() - .route("/health", get(|| async { Json(json!({"status": "ok"})) })) - .route("/v1/models", get(models)); - let base_url = serve(app).await?; - - assert_eq!(preflight(&client()?, &base_url, None).await?, "route-b"); - Ok(()) -} - -#[tokio::test] -async fn preflight_rejects_empty_and_unknown_model() -> TestResult { - let empty = Router::new() - .route("/health", get(|| async { Json(json!({"status": "ok"})) })) - .route("/v1/models", get(|| async { Json(json!({"data": []})) })); - let base_url = serve(empty).await?; - let error = preflight(&client()?, &base_url, None).await.unwrap_err(); - assert!(error.contains("no model to test"), "{error}"); - - let one = Router::new() - .route("/health", get(|| async { Json(json!({"status": "ok"})) })) - .route( - "/v1/models", - get(|| async { Json(json!({"data": [{"id": "route-a"}]})) }), - ); - let base_url = serve(one).await?; - let error = preflight(&client()?, &base_url, Some("missing-model")) - .await - .unwrap_err(); - assert!(error.contains("is not listed"), "{error}"); - Ok(()) -} - -#[tokio::test] -async fn send_request_accepts_json_and_streaming_success() -> TestResult { - let base_url = serve(healthy_switchyard()).await?; - let http = client()?; - - let (error, _) = send_request( - &http, - &base_url, - Endpoint::Chat, - &request_body(Endpoint::Chat, "route", "hi", 8, false), - ) - .await; - assert_eq!(error, None); - let (error, _) = send_request( - &http, - &base_url, - Endpoint::Chat, - &request_body(Endpoint::Chat, "route", "hi", 8, true), - ) - .await; - assert_eq!(error, None); - Ok(()) -} - -#[tokio::test] -async fn send_request_rejects_missing_fields_and_non_sse_stream() -> TestResult { - async fn chat(Json(body): Json) -> Response { - if is_stream(&body) { - // 200 but application/json instead of an event stream. - Json(json!({"choices": []})).into_response() - } else { - // 200 without the required "choices" field. - Json(json!({})).into_response() - } - } - let base_url = serve(Router::new().route("/v1/chat/completions", post(chat))).await?; - let http = client()?; - - let (error, _) = send_request( - &http, - &base_url, - Endpoint::Chat, - &request_body(Endpoint::Chat, "route", "hi", 8, false), - ) - .await; - assert_eq!(error.as_deref(), Some("invalid_response")); - let (error, _) = send_request( - &http, - &base_url, - Endpoint::Chat, - &request_body(Endpoint::Chat, "route", "hi", 8, true), - ) - .await; - assert_eq!(error.as_deref(), Some("invalid_stream")); - Ok(()) -} - -#[tokio::test] -async fn send_request_reports_http_error_and_empty_stream() -> TestResult { - async fn chat(Json(body): Json) -> Response { - if is_stream(&body) { - sse_body("\n\n") // event stream with no data - } else { - (StatusCode::BAD_REQUEST, "bad request").into_response() - } - } - let base_url = serve(Router::new().route("/v1/chat/completions", post(chat))).await?; - let http = client()?; - - let (error, _) = send_request( - &http, - &base_url, - Endpoint::Chat, - &request_body(Endpoint::Chat, "route", "hi", 8, false), - ) - .await; - assert_eq!(error.as_deref(), Some("http_400")); - let (error, _) = send_request( - &http, - &base_url, - Endpoint::Chat, - &request_body(Endpoint::Chat, "route", "hi", 8, true), - ) - .await; - assert_eq!(error.as_deref(), Some("empty_stream")); - Ok(()) -} +async fn bad_responses_and_canary_write_a_failing_summary() -> TestResult { + let (app, _) = switchyard(true); + let server = serve(app).await?; + let dir = tempfile::tempdir()?; + let results_dir = dir.path().join("soak-results"); + let args = args( + &server.base_url, + results_dir.to_str().ok_or("non-utf8 results dir")?, + )?; -#[tokio::test] -async fn read_server_state_treats_non_dict_health_as_unhealthy() -> TestResult { - let app = Router::new() - // Valid JSON, but a list rather than an object. - .route("/health", get(|| async { Json(json!(["ok"])) })) - .route( - "/metrics", - get(|| async { "switchyard_total_requests 5\nswitchyard_total_errors 0\n" }), - ); - let base_url = serve(app).await?; + assert_eq!(run(args).await?, 1); - let (healthy, metrics) = read_server_state(&client()?, &base_url).await; - assert!(!healthy); - assert_eq!(metrics.get("switchyard_total_requests"), Some(&5.0)); + let summary: Value = + serde_json::from_str(&std::fs::read_to_string(results_dir.join("summary.json"))?)?; + assert_eq!(summary["passed"], json!(false)); + assert!( + summary["error_kinds"]["invalid_response"] + .as_u64() + .unwrap_or(0) + > 0 + ); + assert!( + summary["error_kinds"]["invalid_stream"] + .as_u64() + .unwrap_or(0) + > 0 + ); + assert!( + summary["invalid_request_canary_failures"] + .as_u64() + .unwrap_or(0) + > 0 + ); Ok(()) } #[tokio::test] -async fn invalid_request_canary_flags_non_400() -> TestResult { - let app = Router::new() - .route("/health", get(|| async { Json(json!({"status": "ok"})) })) - // Should have been HTTP 400 for an invalid request. - .route( - "/v1/chat/completions", - post(|| async { Json(json!({"choices": []})) }), - ); - let base_url = serve(app).await?; - +async fn unknown_model_is_rejected_before_the_run_starts() -> TestResult { + let (app, _) = switchyard(false); + let server = serve(app).await?; let dir = tempfile::tempdir()?; - let stats = Arc::new(Mutex::new(RunStats::new(0))); - let writer = Arc::new(Mutex::new(ResultsWriter::new(&dir.path().join("results"))?)); - let stop = Arc::new(Stop::new()); - - let task = tokio::spawn(invalid_request_canary( - client()?, - base_url, - 0.05, - "soak-route".to_string(), - stop.clone(), - stats.clone(), - writer.clone(), - )); - tokio::time::sleep(Duration::from_millis(120)).await; - stop.set(); - task.await??; + let results_dir = dir.path().join("soak-results"); + let args = Args::try_parse_from([ + "switchyard-soak", + "--base-url", + &server.base_url, + "--model", + "missing", + "--duration", + "1s", + "--results-dir", + results_dir.to_str().ok_or("non-utf8 results dir")?, + ])?; - let stats = stats.lock(); - assert!(stats.canaries >= 1); - assert_eq!(stats.canary_failures, stats.canaries); + let error = run(args).await.unwrap_err(); + assert!(error.contains("is not listed"), "{error}"); Ok(()) } diff --git a/docs/operations/soak_test.md b/docs/operations/soak_test.md index 111e363ac..e31b24eee 100644 --- a/docs/operations/soak_test.md +++ b/docs/operations/soak_test.md @@ -19,20 +19,11 @@ then confirms the server is still live. ## Prepare the server -Run the exact commit, build, route bundle, backend, and model planned for the +Run the exact commit, build, server config, backend, and model planned for the release. Do not use a development server in front of a different Switchyard build. -For the Python server: - -```bash -uv sync -uv run switchyard --routing-profiles release-routes.yaml -- serve --port 4000 \ - > switchyard-soak.log 2>&1 & -SOAK_SERVER_PID=$! -``` - -For the Rust server and its libsy algorithms: +Start the standalone server and its libsy algorithms: ```bash cargo build --release -p switchyard-server @@ -44,7 +35,7 @@ SOAK_SERVER_PID=$! Wait for `GET http://127.0.0.1:4000/health` to return HTTP 200 with the JSON body `{"status": "ok"}`; the runner requires both before it starts. Check `GET http://127.0.0.1:4000/v1/models` and choose the model id that represents -the release workload. +the release workload. Pass that exact id with `--model`. Run the server and test from a dedicated host, job scheduler, or terminal multiplexer that will stay alive for the full test. Confirm that the host will @@ -59,46 +50,62 @@ release, then run it directly: cargo build --release -p switchyard-soak ``` -## Rehearse against a mock backend +## Rehearse the complete local stack -Rehearse the soak with no live-provider credentials or cost by running -[VidaiMock](https://github.com/vidaiUK/VidaiMock) — an Apache-2.0 mock LLM server that speaks the -OpenAI, Anthropic, and Responses wire formats with realistic streaming — as the backend behind -`switchyard-server`. `scripts/soak-rehearsal.sh` starts the mock, points a passthrough -`switchyard-server` at it, waits for both to report healthy, and runs the soak against the local -stack: +The rehearsal embeds [VidaiMock](https://github.com/vidaiUK/VidaiMock) as a Rust library, so it +does not need a separate `vidaimock` command. Build the server, soak tester, and mock helper from +the commit under test: ```bash -# Requires the vidaimock binary on PATH (github.com/vidaiUK/VidaiMock/releases) and release builds -# of switchyard-server and switchyard-soak. -scripts/soak-rehearsal.sh --duration 5m --concurrency 8 --invalid-canary-interval 0 +cargo build --release -p switchyard-server -p switchyard-soak \ + --bins --example switchyard-soak-mock ``` -Disable the invalid-request canary (`--invalid-canary-interval 0`) against a mock backend: the -canary checks that invalid input is rejected with HTTP 400, which a permissive mock does not do. - -To rehearse the failure gates, degrade the backend and confirm the soak fails closed. For example, -raise the mock's latency past the request timeout and every request times out: +Install [oha](https://github.com/hatoo/oha) and +[NVIDIA AIPerf](https://docs.nvidia.com/aiperf/reference/command-line-options): ```bash -MOCK_LATENCY_MS=2500 scripts/soak-rehearsal.sh --duration 30s --request-timeout 1 \ - --invalid-canary-interval 0 -# Soak FAIL: ... error_rate=100.0000% ... (error_kinds: timeout), exit 1 +cargo install oha +uv tool install aiperf ``` -## Raw throughput with vegeta - -The soak test is closed-loop and validates every response body and stream. For a quick open-loop -capacity figure — a fixed request rate, HTTP status only — [vegeta](https://github.com/tsenart/vegeta) -is a useful companion. It does not validate response bodies or streaming, so it complements the soak -rather than replacing it. +Then run the Python rehearsal from the repository root: ```bash -printf 'POST http://127.0.0.1:4000/v1/chat/completions\nContent-Type: application/json\n@body.json\n' > targets.txt -printf '{"model":"RELEASE_MODEL_ID","messages":[{"role":"user","content":"ping"}],"max_tokens":8}' > body.json -vegeta attack -targets=targets.txt -rate=100 -duration=30s | vegeta report +python3.12 scripts/soak_rehearsal.py --duration 10s --concurrency 4 ``` +`--duration` controls the Rust soak phase. The route smoke, oha, and AIPerf phases are short and +bounded by request count. `--help` explains every rehearsal flag. Set `OHA_BIN`, `AIPERF_BIN`, +`SWITCHYARD_SERVER_BIN`, `SWITCHYARD_SOAK_BIN`, or `SWITCHYARD_SOAK_MOCK_BIN` when a command is not +on `PATH` or not under `target/release`. + +The script checks all five commands before starting a process. A missing command prints a warning, +the build or install command, and the matching environment-variable override. Cargo compiles the +embedded VidaiMock library, but it does not install the unrelated oha or AIPerf executables during +a normal build. + +The script gives each tool one job: + +| Tool | Job in the rehearsal | +|---|---| +| Embedded VidaiMock | Supplies local OpenAI-compatible model responses with configurable latency and no provider cost. | +| oha | Sends raw concurrent HTTP requests through the random route and writes a status and latency distribution. | +| Python route smoke | Sends one Chat Completions request through each route. The stage request includes a critical tool failure so the signal scorer takes its escalation path. | +| AIPerf | Sends streaming Chat Completions through passthrough routing and records LLM and token latency artifacts. | +| `switchyard-soak` | Runs passthrough routing through Chat Completions, Messages, and Responses, with streaming on and off, while checking liveness, metrics, process sampling, and result gates. | + +The checked-in rehearsal config exercises `noop`, `random`, `passthrough`, `llm_classifier`, and +`stage_router`. It uses the accepted maximum retry count (10), a zero-weight random target, and the +upper classifier and stage thresholds (1.0). The config is validated with +`switchyard-server --dry-run` before either service starts. + +The runner does not invent operational maxima for unbounded values such as target count or recent +turn window. Focused Rust tests cover the real bounded structures at cap plus one: 1,024 routing +overflow identities, 4,096 affinity assignments, 100,000 retained latency samples, and 10,000 +recorded error details. Server tests cover invalid thresholds, malformed classifier verdicts, +missing subagent identifiers, target failures, and the retry value immediately above the maximum. + ## Run the 48-hour test Choose concurrency from the release capacity plan. Increase it in short @@ -177,18 +184,12 @@ Each run creates a timestamped directory under `soak-results/`: read it as a long-run average rather than a spike detector. - `errors.jsonl` records up to 10,000 request and canary failures. - `summary.json` records the final pass result and any failed gates. -- `status.json` is overwritten atomically each interval with a one-object live snapshot — progress - toward the target duration, cumulative requests, cumulative error rate, health, detected restarts, - and a `status` of `ok`, `degraded`, or `stalled`. Read it (or tail the run log, whose interval line - carries the same fields and status token) to monitor a run in progress; on a remote host, - `cat /status.json` gives an at-a-glance confidence check without parsing `intervals.csv`. -When the run ends the command also prints a latency percentile table (p50 through -p99.9) and an ASCII latency histogram to the terminal, for a quick read of the -distribution without opening the result files. +Tail the run log to monitor progress, cumulative error rate, health, RSS, and the +`OK`, `DEGRADED`, or `STALLED` interval status while the test runs. Before approving the release, check `intervals.csv` for late failures, falling throughput, increasing p95 or p99 latency, and steady RSS growth. Compare the first and last several hours, not only the run-wide averages. Attach `summary.json`, the interval chart, the Switchyard log, the tested commit, and -the route bundle to the release record. +the server config to the release record. diff --git a/scripts/soak-rehearsal.sh b/scripts/soak-rehearsal.sh deleted file mode 100755 index 8da420bb9..000000000 --- a/scripts/soak-rehearsal.sh +++ /dev/null @@ -1,87 +0,0 @@ -#!/usr/bin/env bash -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Hermetic soak rehearsal: run the soak tester against switchyard-server backed by VidaiMock -# (https://github.com/vidaiUK/VidaiMock), a mock LLM backend, with no live provider credentials -# or cost. Arguments are passed through to switchyard-soak; with none, a short rehearsal runs with -# the invalid-request canary disabled, because a permissive mock does not reject invalid input. -# -# Requires the vidaimock binary on PATH (or set VIDAIMOCK_BIN) and release builds of -# switchyard-server and switchyard-soak (cargo build --release -p switchyard-server -p switchyard-soak). -set -euo pipefail - -REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -VIDAIMOCK_BIN="${VIDAIMOCK_BIN:-vidaimock}" -SERVER_BIN="${SWITCHYARD_SERVER_BIN:-$REPO_ROOT/target/release/switchyard-server}" -SOAK_BIN="${SWITCHYARD_SOAK_BIN:-$REPO_ROOT/target/release/switchyard-soak}" -MOCK_PORT="${MOCK_PORT:-8100}" -SERVER_PORT="${SERVER_PORT:-4000}" -MOCK_LATENCY_MS="${MOCK_LATENCY_MS:-40}" - -command -v "$VIDAIMOCK_BIN" >/dev/null 2>&1 || { - echo "error: '$VIDAIMOCK_BIN' not found on PATH." >&2 - echo "Install VidaiMock from https://github.com/vidaiUK/VidaiMock/releases, or set VIDAIMOCK_BIN." >&2 - exit 1 -} -for bin in "$SERVER_BIN" "$SOAK_BIN"; do - [ -x "$bin" ] || { - echo "error: '$bin' not found; run: cargo build --release -p switchyard-server -p switchyard-soak" >&2 - exit 1 - } -done - -workdir="$(mktemp -d)" -mock_pid="" -server_pid="" -cleanup() { - [ -n "$server_pid" ] && kill "$server_pid" 2>/dev/null || true - [ -n "$mock_pid" ] && kill "$mock_pid" 2>/dev/null || true - rm -rf "$workdir" -} -trap cleanup EXIT - -wait_health() { # $1=url $2=name - for _ in $(seq 1 60); do - curl -sf "$1" >/dev/null 2>&1 && return 0 - sleep 0.25 - done - echo "error: $2 did not become healthy at $1" >&2 - return 1 -} - -cat >"$workdir/mock.toml" <"$workdir/vidaimock.log" 2>&1 & -mock_pid=$! -disown 2>/dev/null || true # drop job control so cleanup does not print "Terminated" -wait_health "http://127.0.0.1:${MOCK_PORT}/health" "VidaiMock" - -echo "Starting switchyard-server on :${SERVER_PORT}" -"$SERVER_BIN" --config "$workdir/mock.toml" --port "$SERVER_PORT" \ - >"$workdir/switchyard-server.log" 2>&1 & -server_pid=$! -disown 2>/dev/null || true -wait_health "http://127.0.0.1:${SERVER_PORT}/health" "switchyard-server" - -if [ "$#" -gt 0 ]; then - "$SOAK_BIN" --base-url "http://127.0.0.1:${SERVER_PORT}" --model switchyard/mock "$@" -else - "$SOAK_BIN" --base-url "http://127.0.0.1:${SERVER_PORT}" --model switchyard/mock \ - --duration 60s --concurrency 8 --stream-ratio 0.5 --report-interval 10 \ - --invalid-canary-interval 0 --server-pid "$server_pid" -fi diff --git a/scripts/soak_rehearsal.py b/scripts/soak_rehearsal.py new file mode 100755 index 000000000..75b7b061a --- /dev/null +++ b/scripts/soak_rehearsal.py @@ -0,0 +1,481 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Rehearse every Switchyard route with embedded VidaiMock, oha, AIPerf, and the soak test.""" + +import argparse +import json +import os +import shutil +import subprocess +import sys +import time +import urllib.error +import urllib.request +from collections.abc import Sequence +from dataclasses import dataclass +from datetime import UTC, datetime +from pathlib import Path +from typing import TextIO + +ROUTES = ( + "switchyard/noop", + "switchyard/random", + "switchyard/passthrough", + "switchyard/classifier", + "switchyard/stage", +) +MOCK_PORT = 8100 +OHA_REQUESTS = 100 +AIPERF_REQUESTS = 20 + + +@dataclass +class Child: + """A child process and the log file that must stay open while it runs.""" + + name: str + process: subprocess.Popen[str] + log: TextIO + log_path: Path + + +@dataclass(frozen=True) +class RequiredBinary: + """One command the rehearsal must find before starting any process.""" + + label: str + env_var: str + value: str + setup: str + + +def positive_int(value: str) -> int: + """Parse a command-line integer that must be greater than zero.""" + parsed = int(value) + if parsed <= 0: + raise argparse.ArgumentTypeError("must be greater than zero") + return parsed + + +def nonnegative_int(value: str) -> int: + """Parse a command-line integer that may be zero.""" + parsed = int(value) + if parsed < 0: + raise argparse.ArgumentTypeError("must be zero or greater") + return parsed + + +def tcp_port(value: str) -> int: + """Parse a valid nonzero TCP port.""" + parsed = positive_int(value) + if parsed > 65_535: + raise argparse.ArgumentTypeError("must be between 1 and 65535") + return parsed + + +def parser() -> argparse.ArgumentParser: + """Build the plain-English command-line interface.""" + command = argparse.ArgumentParser( + description=( + "Start a local VidaiMock-backed Switchyard stack, smoke every route, then run oha, " + "AIPerf, and switchyard-soak." + ), + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + command.add_argument( + "--duration", + default="10s", + help="time to run switchyard-soak; use an s, m, or h suffix", + ) + command.add_argument( + "--concurrency", + type=positive_int, + default=4, + help="concurrent requests used by oha, AIPerf, and the soak run", + ) + command.add_argument( + "--mock-latency-ms", + type=nonnegative_int, + default=40, + help="response latency VidaiMock adds to each backend call", + ) + command.add_argument( + "--server-port", + type=tcp_port, + default=4000, + help="local TCP port used by switchyard-server", + ) + command.add_argument( + "--output-dir", + type=Path, + help="new directory for generated config, logs, and tool results", + ) + return command + + +def find_binary(value: str) -> str | None: + """Return an executable path, or None when the command cannot run.""" + if os.sep in value: + path = Path(value).expanduser().resolve() + if path.is_file() and os.access(path, os.X_OK): + return str(path) + else: + found = shutil.which(value) + if found is not None: + return found + return None + + +def resolve_binaries(requirements: Sequence[RequiredBinary]) -> dict[str, str]: + """Resolve every command and print all missing-command setup instructions.""" + resolved = {} + missing = [] + for requirement in requirements: + path = find_binary(requirement.value) + if path is None: + missing.append(requirement) + else: + resolved[requirement.label] = path + + for requirement in missing: + print( + f"warning: {requirement.label} executable not found: {requirement.value}", + file=sys.stderr, + ) + print(f" {requirement.setup}", file=sys.stderr) + print( + f" Or set {requirement.env_var}=/path/to/{Path(requirement.value).name}", + file=sys.stderr, + ) + if missing: + raise RuntimeError(f"install or build the {len(missing)} missing command(s), then rerun") + return resolved + + +def start_child(name: str, command: Sequence[str], log_path: Path) -> Child: + """Start one long-running process and retain its combined log.""" + print(f"Starting {name}; log: {log_path}") + log = log_path.open("w", encoding="utf-8") + try: + process = subprocess.Popen( + list(command), + stdout=log, + stderr=subprocess.STDOUT, + text=True, + ) + except OSError: + log.close() + raise + return Child(name=name, process=process, log=log, log_path=log_path) + + +def stop_child(child: Child) -> None: + """Stop one child and close its log, escalating to kill after five seconds.""" + if child.process.poll() is None: + child.process.terminate() + try: + child.process.wait(timeout=5) + except subprocess.TimeoutExpired: + child.process.kill() + child.process.wait(timeout=5) + child.log.close() + + +def wait_for_health(child: Child, url: str, expected_status: str | None = None) -> None: + """Wait up to 30 seconds for a child process's health endpoint.""" + for _ in range(120): + if child.process.poll() is not None: + raise RuntimeError( + f"{child.name} exited with status {child.process.returncode}; see {child.log_path}" + ) + try: + with urllib.request.urlopen(url, timeout=1) as response: + body = response.read() + document = json.loads(body) if expected_status is not None else None + status = document.get("status") if isinstance(document, dict) else None + if response.status == 200 and ( + expected_status is None or status == expected_status + ): + return + except (OSError, urllib.error.URLError, json.JSONDecodeError): + pass + time.sleep(0.25) + raise RuntimeError(f"{child.name} did not become healthy at {url}; see {child.log_path}") + + +def run_checked(name: str, command: Sequence[str], log_path: Path) -> None: + """Run one finite tool and keep its output in a named log file.""" + print(f"Running {name}; log: {log_path}") + with log_path.open("w", encoding="utf-8") as log: + result = subprocess.run( + list(command), + stdout=log, + stderr=subprocess.STDOUT, + text=True, + check=False, + ) + if result.returncode != 0: + raise RuntimeError(f"{name} failed with status {result.returncode}; see {log_path}") + + +def smoke_routes(base_url: str, output_path: Path) -> None: + """Send one production HTTP request through every configured route.""" + ordinary = [{"role": "user", "content": "Reply with exactly OK."}] + stage_edge = [ + {"role": "user", "content": "Fix the build."}, + { + "role": "assistant", + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": {"name": "Bash", "arguments": '{"command":"cargo test"}'}, + } + ], + }, + { + "role": "tool", + "tool_call_id": "call_1", + "content": "fatal runtime error: out of memory", + }, + ] + records = [] + for route in ROUTES: + body = json.dumps( + { + "model": route, + "messages": stage_edge if route == "switchyard/stage" else ordinary, + "max_tokens": 8, + "stream": False, + } + ).encode() + request = urllib.request.Request( + f"{base_url}/v1/chat/completions", + data=body, + headers={"content-type": "application/json"}, + method="POST", + ) + with urllib.request.urlopen(request, timeout=30) as response: + records.append( + { + "route": route, + "status": response.status, + "selected_model": response.headers.get("x-model-router-selected-model"), + "response": json.loads(response.read()), + } + ) + output_path.write_text( + "".join(f"{json.dumps(record)}\n" for record in records), encoding="utf-8" + ) + + +def default_output_dir(repo_root: Path) -> Path: + """Choose a new timestamped directory under the repository.""" + stamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ") + return repo_root / "soak-rehearsal-results" / stamp + + +def run_rehearsal(args: argparse.Namespace) -> Path: + """Run the complete local rehearsal and return its output directory.""" + repo_root = Path(__file__).resolve().parents[1] + rust_build = ( + "Build the Rust commands with: cargo build --release -p switchyard-server " + "-p switchyard-soak --bins --example switchyard-soak-mock" + ) + binaries = resolve_binaries( + ( + RequiredBinary( + "switchyard-server", + "SWITCHYARD_SERVER_BIN", + os.environ.get( + "SWITCHYARD_SERVER_BIN", + str(repo_root / "target/release/switchyard-server"), + ), + rust_build, + ), + RequiredBinary( + "switchyard-soak", + "SWITCHYARD_SOAK_BIN", + os.environ.get( + "SWITCHYARD_SOAK_BIN", str(repo_root / "target/release/switchyard-soak") + ), + rust_build, + ), + RequiredBinary( + "embedded VidaiMock helper", + "SWITCHYARD_SOAK_MOCK_BIN", + os.environ.get( + "SWITCHYARD_SOAK_MOCK_BIN", + str(repo_root / "target/release/examples/switchyard-soak-mock"), + ), + rust_build, + ), + RequiredBinary( + "oha", + "OHA_BIN", + os.environ.get("OHA_BIN", "oha"), + "Install oha with: cargo install oha", + ), + RequiredBinary( + "AIPerf", + "AIPERF_BIN", + os.environ.get("AIPERF_BIN", "aiperf"), + "Install AIPerf with: uv tool install aiperf", + ), + ) + ) + server_bin = binaries["switchyard-server"] + soak_bin = binaries["switchyard-soak"] + mock_bin = binaries["embedded VidaiMock helper"] + oha_bin = binaries["oha"] + aiperf_bin = binaries["AIPerf"] + + output_dir = (args.output_dir or default_output_dir(repo_root)).resolve() + output_dir.mkdir(parents=True, exist_ok=False) + config_path = output_dir / "routes.toml" + shutil.copy2(repo_root / "scripts/soak_rehearsal.toml", config_path) + + run_checked( + "switchyard-server config validation", + [server_bin, "--config", str(config_path), "--dry-run"], + output_dir / "config-validation.log", + ) + + children: list[Child] = [] + try: + mock = start_child( + "embedded VidaiMock", + [ + mock_bin, + "--port", + str(MOCK_PORT), + "--latency-ms", + str(args.mock_latency_ms), + ], + output_dir / "vidaimock.log", + ) + children.append(mock) + wait_for_health(mock, f"http://127.0.0.1:{MOCK_PORT}/health") + + server = start_child( + "switchyard-server", + [server_bin, "--config", str(config_path), "--port", str(args.server_port)], + output_dir / "switchyard-server.log", + ) + children.append(server) + base_url = f"http://127.0.0.1:{args.server_port}" + wait_for_health(server, f"{base_url}/health", "ok") + smoke_routes(base_url, output_dir / "route-smoke.jsonl") + + oha_body = output_dir / "oha-request.json" + oha_body.write_text( + json.dumps( + { + "model": "switchyard/random", + "messages": [{"role": "user", "content": "Reply with exactly OK."}], + "max_tokens": 8, + "stream": False, + } + ), + encoding="utf-8", + ) + run_checked( + "oha raw HTTP load", + [ + oha_bin, + "-n", + str(OHA_REQUESTS), + "-c", + str(args.concurrency), + "--no-tui", + "--method", + "POST", + "-T", + "application/json", + "-D", + str(oha_body), + "--output-format", + "json", + "--output", + str(output_dir / "oha.json"), + f"{base_url}/v1/chat/completions", + ], + output_dir / "oha.log", + ) + + run_checked( + "AIPerf streaming profile", + [ + aiperf_bin, + "profile", + "--model", + "switchyard/passthrough", + "--url", + base_url, + "--endpoint-type", + "chat", + "--streaming", + "--tokenizer", + "builtin", + "--use-legacy-max-tokens", + "--isl", + "32", + "--osl", + "8", + "--concurrency", + str(args.concurrency), + "--request-count", + str(AIPERF_REQUESTS), + "--request-timeout-seconds", + "30", + "--artifact-dir", + str(output_dir / "aiperf"), + ], + output_dir / "aiperf.log", + ) + + run_checked( + "switchyard-soak endpoint and streaming matrix", + [ + soak_bin, + "--base-url", + base_url, + "--model", + "switchyard/passthrough", + "--duration", + args.duration, + "--concurrency", + str(args.concurrency), + "--report-interval", + "2", + "--invalid-canary-interval", + "0", + "--server-pid", + str(server.process.pid), + "--results-dir", + str(output_dir / "soak"), + ], + output_dir / "soak.log", + ) + finally: + for child in reversed(children): + stop_child(child) + + return output_dir + + +def main(argv: Sequence[str] | None = None) -> int: + """Parse arguments, run the rehearsal, and print one actionable failure.""" + args = parser().parse_args(argv) + try: + output_dir = run_rehearsal(args) + except (OSError, RuntimeError) as error: + print(f"soak rehearsal failed: {error}", file=sys.stderr) + return 1 + print(f"Soak rehearsal passed; results: {output_dir}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/soak_rehearsal.toml b/scripts/soak_rehearsal.toml new file mode 100644 index 000000000..5192d7e87 --- /dev/null +++ b/scripts/soak_rehearsal.toml @@ -0,0 +1,57 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version = 1 + +[llm_clients.mock] +format = "openai_chat" +base_url = "http://127.0.0.1:8100/v1" +max_retries = 10 + +[targets.classifier] +id = "mock/classifier" +llm_client = "mock" + +[targets.strong] +id = "mock/strong" +llm_client = "mock" + +[targets.weak] +id = "mock/weak" +llm_client = "mock" + +[routes.noop] +id = "switchyard/noop" +type = "noop" + +[routes.random] +id = "switchyard/random" +type = "random" +targets = ["weak", "strong"] +weights = [1, 0] +seed = 42 + +[routes.passthrough] +id = "switchyard/passthrough" +type = "passthrough" +target = "weak" + +[routes.classifier] +id = "switchyard/classifier" +type = "llm_classifier" +mode = "capability" +classifier_target = "classifier" +strong_target = "strong" +weak_target = "weak" +base_threshold = 1.0 +session_affinity = true +message_hash_fallback = true +max_output_tokens = 1 + +[routes.stage] +id = "switchyard/stage" +type = "stage_router" +capable_target = "strong" +efficient_target = "weak" +picker = "efficient_first" +confidence_threshold = 1.0 From a3358ca330a20745a723268899d05efb6107e1e1 Mon Sep 17 00:00:00 2001 From: Elyas Mehtabuddin Date: Tue, 18 Aug 2026 02:42:46 +0000 Subject: [PATCH 3/9] fix(operations): harden local soak rehearsals Signed-off-by: Elyas Mehtabuddin --- .gitignore | 1 + crates/libsy/src/algorithms/noop.rs | 30 +++++++++++++++++-- crates/switchyard-soak/src/report.rs | 2 +- crates/switchyard-soak/tests/soak.rs | 11 +++---- .../src/codecs/anthropic/buffered.rs | 8 ++++- .../src/codecs/openai_chat/buffered.rs | 8 ++++- .../tests/request_translation.rs | 12 ++++++++ scripts/soak_rehearsal.py | 4 +-- tests/test_soak_rehearsal.py | 20 +++++++++++++ 9 files changed, 84 insertions(+), 12 deletions(-) create mode 100644 tests/test_soak_rehearsal.py diff --git a/.gitignore b/.gitignore index ec398cb0b..c1457813b 100644 --- a/.gitignore +++ b/.gitignore @@ -151,6 +151,7 @@ plans/ # Benchmark result directories bench_results*/ soak-results/ +soak-rehearsal-results/ jobs/**/*.* pytest-of-*/ diff --git a/crates/libsy/src/algorithms/noop.rs b/crates/libsy/src/algorithms/noop.rs index 95f4c7212..8a9e42c47 100644 --- a/crates/libsy/src/algorithms/noop.rs +++ b/crates/libsy/src/algorithms/noop.rs @@ -23,11 +23,12 @@ impl Algorithm for Noop { } async fn route(self: Arc, _driver: Driver, request: Request) -> Result { + let streaming = request.llm_request.stream; let model_id = request .model_id() .unwrap_or_else(|| ModelId::from("switchyard/noop")); tracing::info!(target = %model_id, "noop returned its synthetic response"); - let llm_response = LlmResponse::Agg(AggLlmResponse { + let aggregate = AggLlmResponse { id: Some("switchyard-noop".to_string()), model: Some(model_id.to_string()), outputs: vec![ResponseOutput { @@ -38,7 +39,12 @@ impl Algorithm for Noop { stop_reason: Some(StopReason::EndTurn), }], ..Default::default() - }); + }; + let llm_response = if streaming { + LlmResponse::Stream(aggregate.into_stream()) + } else { + LlmResponse::Agg(aggregate) + }; let response = Response { llm_response, metadata: request.metadata.clone(), @@ -75,4 +81,24 @@ mod tests { assert_eq!(response.selected_model(), Some(TEST_MODEL)); Ok(()) } + + #[tokio::test] + async fn test_noop_algo_streams_when_requested() -> Result<()> { + let request = Request { + llm_request: LlmRequest { + model: Some("streaming-noop".to_string()), + messages: vec![Message::text(Role::User, "hi")], + stream: true, + ..LlmRequest::default() + }, + raw_request: None, + metadata: None, + }; + + let algorithm: Arc = Arc::new(Noop {}); + let (_, response) = test_drive(algorithm, request, echo()).await?; + + assert!(matches!(response.llm_response, LlmResponse::Stream(_))); + Ok(()) + } } diff --git a/crates/switchyard-soak/src/report.rs b/crates/switchyard-soak/src/report.rs index b941e4ea5..69e512528 100644 --- a/crates/switchyard-soak/src/report.rs +++ b/crates/switchyard-soak/src/report.rs @@ -268,7 +268,7 @@ pub async fn invalid_request_canary( let invalid = context .client .post(format!("{}/v1/chat/completions", context.base_url)) - .json(&json!({"model": workload.model, "messages": []})) + .json(&json!({"model": workload.model, "messages": "invalid"})) .send() .await?; let health = context diff --git a/crates/switchyard-soak/tests/soak.rs b/crates/switchyard-soak/tests/soak.rs index 74fd288ed..8db51aa6e 100644 --- a/crates/switchyard-soak/tests/soak.rs +++ b/crates/switchyard-soak/tests/soak.rs @@ -67,11 +67,12 @@ fn switchyard(is_chat_broken: bool) -> (Router, Arc>>) { Json(json!({"data": [{"id": "soak-route"}]})).into_response() } async fn chat(State(state): State, Json(body): Json) -> Response { - let empty = body - .get("messages") - .and_then(Value::as_array) - .is_some_and(Vec::is_empty); - if empty { + let invalid_messages = body.get("messages").is_some_and(|messages| { + messages + .as_array() + .is_none_or(|messages| messages.is_empty()) + }); + if invalid_messages { return if state.is_chat_broken { Json(json!({"choices": []})).into_response() } else { diff --git a/crates/switchyard-translation/src/codecs/anthropic/buffered.rs b/crates/switchyard-translation/src/codecs/anthropic/buffered.rs index 87dc58b6c..23fd5140c 100644 --- a/crates/switchyard-translation/src/codecs/anthropic/buffered.rs +++ b/crates/switchyard-translation/src/codecs/anthropic/buffered.rs @@ -91,7 +91,13 @@ impl FormatCodec for AnthropicMessagesCodec { content, }); } - if let Some(messages) = body.get("messages").and_then(Value::as_array) { + if let Some(messages) = body.get("messages") { + let messages = messages + .as_array() + .ok_or_else(|| TranslationError::InvalidType { + path: "$.messages".to_string(), + expected: "array", + })?; let mut generated_id = 0; for (index, message) in messages.iter().enumerate() { let Some(message) = message.as_object() else { diff --git a/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs b/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs index d829ca316..4b0394f43 100644 --- a/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs +++ b/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs @@ -72,7 +72,13 @@ impl FormatCodec for OpenAiChatCodec { ..LlmRequest::default() }; - if let Some(messages) = body.get("messages").and_then(Value::as_array) { + if let Some(messages) = body.get("messages") { + let messages = messages + .as_array() + .ok_or_else(|| TranslationError::InvalidType { + path: "$.messages".to_string(), + expected: "array", + })?; let mut generated_id = 0; for (index, message) in messages.iter().enumerate() { let Some(message) = message.as_object() else { diff --git a/crates/switchyard-translation/tests/request_translation.rs b/crates/switchyard-translation/tests/request_translation.rs index e689e5b21..c754994ce 100644 --- a/crates/switchyard-translation/tests/request_translation.rs +++ b/crates/switchyard-translation/tests/request_translation.rs @@ -1782,6 +1782,18 @@ fn malformed_request_fields_are_rejected() { json!({"model": "claude", "max_tokens": "8", "messages": []}), "invalid value at $.max_tokens: expected a non-negative integer", ), + ( + "OpenAI Chat string messages", + WireFormat::OpenAiChat, + json!({"model": "gpt", "messages": "invalid"}), + "expected array at $.messages", + ), + ( + "Anthropic string messages", + WireFormat::AnthropicMessages, + json!({"model": "claude", "max_tokens": 8, "messages": "invalid"}), + "expected array at $.messages", + ), ( "Responses boolean input", WireFormat::OpenAiResponses, diff --git a/scripts/soak_rehearsal.py b/scripts/soak_rehearsal.py index 75b7b061a..7a2ca63b8 100755 --- a/scripts/soak_rehearsal.py +++ b/scripts/soak_rehearsal.py @@ -14,7 +14,7 @@ import urllib.request from collections.abc import Sequence from dataclasses import dataclass -from datetime import UTC, datetime +from datetime import datetime, timezone from pathlib import Path from typing import TextIO @@ -272,7 +272,7 @@ def smoke_routes(base_url: str, output_path: Path) -> None: def default_output_dir(repo_root: Path) -> Path: """Choose a new timestamped directory under the repository.""" - stamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ") + stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") return repo_root / "soak-rehearsal-results" / stamp diff --git a/tests/test_soak_rehearsal.py b/tests/test_soak_rehearsal.py new file mode 100644 index 000000000..8be8b8854 --- /dev/null +++ b/tests/test_soak_rehearsal.py @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import subprocess +import sys +from pathlib import Path + + +def test_help_runs_on_supported_python() -> None: + script = Path(__file__).resolve().parents[1] / "scripts" / "soak_rehearsal.py" + + result = subprocess.run( + [sys.executable, str(script), "--help"], + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode == 0, result.stderr + assert "--mock-latency-ms" in result.stdout From 5112428c96cd5554cf49f7cbcb308a2731f30ae5 Mon Sep 17 00:00:00 2001 From: Elyas Mehtabuddin Date: Tue, 18 Aug 2026 09:51:30 -0700 Subject: [PATCH 4/9] refactor(operations): rename local soak test Signed-off-by: Elyas Mehtabuddin --- .gitignore | 2 +- crates/switchyard-soak/README.md | 48 ++++--------------- .../switchyard-soak/examples/mock_server.rs | 6 +-- crates/switchyard-soak/src/lib.rs | 2 +- docs/operations/soak_test.md | 45 +++++++++-------- ...ak_rehearsal.toml => local_soak_test.toml} | 2 + ...ak_rehearsal.py => run_local_soak_test.py} | 32 ++++++------- tests/test_soak_rehearsal.py | 20 -------- 8 files changed, 54 insertions(+), 103 deletions(-) rename scripts/{soak_rehearsal.toml => local_soak_test.toml} (92%) rename scripts/{soak_rehearsal.py => run_local_soak_test.py} (92%) delete mode 100644 tests/test_soak_rehearsal.py diff --git a/.gitignore b/.gitignore index c1457813b..03c031b23 100644 --- a/.gitignore +++ b/.gitignore @@ -151,7 +151,7 @@ plans/ # Benchmark result directories bench_results*/ soak-results/ -soak-rehearsal-results/ +local-soak-test-results/ jobs/**/*.* pytest-of-*/ diff --git a/crates/switchyard-soak/README.md b/crates/switchyard-soak/README.md index 0134236c1..ec10a7d45 100644 --- a/crates/switchyard-soak/README.md +++ b/crates/switchyard-soak/README.md @@ -69,47 +69,17 @@ The inference workers use a fixed six-case cycle: three public endpoint formats streaming on and off. A deterministic cycle makes missing endpoint or streaming coverage visible in short runs. -## Rehearse the whole toolchain +## Check the local server and load tools -The repository's `scripts/soak_rehearsal.py` starts an embedded VidaiMock server and -`switchyard-server`, validates a configuration containing every server route type, sends one -production HTTP request through each route, then runs three complementary clients: +`scripts/run_local_soak_test.py` starts an embedded VidaiMock server and `switchyard-server`, sends +one HTTP request through each configured route, then runs `oha`, NVIDIA AIPerf, and +`switchyard-soak`. VidaiMock returns fixed local responses, so this local test needs no provider key +and incurs no inference cost. -- `oha` checks raw HTTP concurrency and status-code distribution. -- NVIDIA AIPerf measures LLM request, token, and streaming latency through passthrough routing. -- `switchyard-soak` checks all three API formats, streaming framing, liveness, metrics, process - sampling, result files, and pass/fail gates through the same passthrough route. - -`switchyard-soak` itself does not require VidaiMock, oha, or AIPerf. The local rehearsal builds -VidaiMock's Rust library into a helper command, so it does not need a separate `vidaimock` install. -Build the Rust commands and helper: - -```bash -cargo build --release -p switchyard-server -p switchyard-soak \ - --bins --example switchyard-soak-mock -``` - -Install the two external load generators: - -```bash -cargo install oha -uv tool install aiperf -``` - -Then run the local rehearsal: - -```bash -python3.12 scripts/soak_rehearsal.py --duration 10s --concurrency 4 -``` - -The script checks every required command before it starts a server. When a command is missing, it -prints a warning with the build or install command and the environment variable that can point to -an existing executable. Cargo dependencies compile Rust libraries; a normal `cargo build` does -not install unrelated host commands, so oha and AIPerf remain explicit installs. - -The embedded VidaiMock server supplies deterministic local model responses, so this rehearsal needs no provider key -and incurs no inference cost. It is a preflight check, not a replacement for the 48-hour run -against the release deployment. +The script needs the built Rust programs plus installed `oha` and AIPerf commands. +`switchyard-soak` itself does not require those extra programs and can run by itself against a +release server. See the [operations guide](../../docs/operations/soak_test.md) for setup, the local +command, the 48-hour release run, and result review. ## Results diff --git a/crates/switchyard-soak/examples/mock_server.rs b/crates/switchyard-soak/examples/mock_server.rs index 36d712505..b4231302c 100644 --- a/crates/switchyard-soak/examples/mock_server.rs +++ b/crates/switchyard-soak/examples/mock_server.rs @@ -1,18 +1,18 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Local VidaiMock server used by `scripts/soak_rehearsal.py`. +//! Local VidaiMock server used by `scripts/run_local_soak_test.py`. use std::process::ExitCode; use clap::Parser; use vidaimock::MockServer; -/// Start the embedded mock backend used by the local soak rehearsal. +/// Start the embedded mock backend used by the local soak test. #[derive(Parser)] #[command( name = "switchyard-soak-mock", - about = "Start VidaiMock for the local Switchyard soak rehearsal", + about = "Start VidaiMock for the local Switchyard soak test", after_long_help = "Example:\n cargo run --release -p switchyard-soak --example switchyard-soak-mock -- --port 8100 --latency-ms 40", version )] diff --git a/crates/switchyard-soak/src/lib.rs b/crates/switchyard-soak/src/lib.rs index 6480e5f02..d194cd19e 100644 --- a/crates/switchyard-soak/src/lib.rs +++ b/crates/switchyard-soak/src/lib.rs @@ -33,7 +33,7 @@ use crate::stats::{ #[command( name = "switchyard-soak", about = "Run a sustained, closed-loop load test against a live Switchyard server", - after_long_help = "Examples:\n switchyard-soak --model switchyard/general --duration 5m --concurrency 4\n switchyard-soak --model switchyard/general --duration 48h --server-pid 1234 --max-rss-growth-mib 512\n\nThis command does not require VidaiMock, oha, or AIPerf. The optional scripts/soak_rehearsal.py command uses an embedded VidaiMock helper and warns before it starts when oha or AIPerf is missing.", + after_long_help = "Examples:\n switchyard-soak --model switchyard/general --duration 5m --concurrency 4\n switchyard-soak --model switchyard/general --duration 48h --server-pid 1234 --max-rss-growth-mib 512\n\nThis command does not require VidaiMock, oha, or AIPerf. The optional scripts/run_local_soak_test.py command uses an embedded VidaiMock helper and warns before it starts when oha or AIPerf is missing.", version )] pub struct Args { diff --git a/docs/operations/soak_test.md b/docs/operations/soak_test.md index e31b24eee..2780ad815 100644 --- a/docs/operations/soak_test.md +++ b/docs/operations/soak_test.md @@ -50,11 +50,11 @@ release, then run it directly: cargo build --release -p switchyard-soak ``` -## Rehearse the complete local stack +## Check the local server and load tools -The rehearsal embeds [VidaiMock](https://github.com/vidaiUK/VidaiMock) as a Rust library, so it -does not need a separate `vidaimock` command. Build the server, soak tester, and mock helper from -the commit under test: +The local test embeds [VidaiMock](https://github.com/vidaiUK/VidaiMock) as a Rust library, so it does +not need a separate `vidaimock` command. Build the server, soak tester, and mock helper from the +commit under test: ```bash cargo build --release -p switchyard-server -p switchyard-soak \ @@ -69,47 +69,46 @@ cargo install oha uv tool install aiperf ``` -Then run the Python rehearsal from the repository root: +Then run the local test from the repository root: ```bash -python3.12 scripts/soak_rehearsal.py --duration 10s --concurrency 4 +python3.12 scripts/run_local_soak_test.py --duration 10s --concurrency 4 ``` -`--duration` controls the Rust soak phase. The route smoke, oha, and AIPerf phases are short and -bounded by request count. `--help` explains every rehearsal flag. Set `OHA_BIN`, `AIPERF_BIN`, +`--duration` controls how long the Rust soak tester runs. The route checks, oha, and AIPerf runs are +short and limited by request count. `--help` explains every flag. Set `OHA_BIN`, `AIPERF_BIN`, `SWITCHYARD_SERVER_BIN`, `SWITCHYARD_SOAK_BIN`, or `SWITCHYARD_SOAK_MOCK_BIN` when a command is not on `PATH` or not under `target/release`. The script checks all five commands before starting a process. A missing command prints a warning, -the build or install command, and the matching environment-variable override. Cargo compiles the -embedded VidaiMock library, but it does not install the unrelated oha or AIPerf executables during -a normal build. +the build or install command, and the matching environment variable. Cargo compiles the embedded +VidaiMock library, but it does not install the unrelated oha or AIPerf programs during a normal +build. The script gives each tool one job: -| Tool | Job in the rehearsal | +| Tool | Job in the local test | |---|---| | Embedded VidaiMock | Supplies local OpenAI-compatible model responses with configurable latency and no provider cost. | | oha | Sends raw concurrent HTTP requests through the random route and writes a status and latency distribution. | -| Python route smoke | Sends one Chat Completions request through each route. The stage request includes a critical tool failure so the signal scorer takes its escalation path. | -| AIPerf | Sends streaming Chat Completions through passthrough routing and records LLM and token latency artifacts. | -| `switchyard-soak` | Runs passthrough routing through Chat Completions, Messages, and Responses, with streaming on and off, while checking liveness, metrics, process sampling, and result gates. | +| Python route checks | Sends one Chat Completions request through each route. The stage request includes a critical tool failure so the signal scorer selects the capable target. | +| AIPerf | Sends streaming Chat Completions through passthrough routing and records LLM, token, and response-time results. | +| `switchyard-soak` | Runs passthrough routing through Chat Completions, Messages, and Responses, with streaming on and off, while checking server health, metrics, process use, and required results. | -The checked-in rehearsal config exercises `noop`, `random`, `passthrough`, `llm_classifier`, and +`scripts/local_soak_test.toml` exercises `noop`, `random`, `passthrough`, `llm_classifier`, and `stage_router`. It uses the accepted maximum retry count (10), a zero-weight random target, and the upper classifier and stage thresholds (1.0). The config is validated with `switchyard-server --dry-run` before either service starts. -The runner does not invent operational maxima for unbounded values such as target count or recent -turn window. Focused Rust tests cover the real bounded structures at cap plus one: 1,024 routing -overflow identities, 4,096 affinity assignments, 100,000 retained latency samples, and 10,000 -recorded error details. Server tests cover invalid thresholds, malformed classifier verdicts, -missing subagent identifiers, target failures, and the retry value immediately above the maximum. +The runner does not add limits for target count or recent-turn history because the server does not +limit them. Rust tests cover the real limits: 4,096 saved route assignments, 100,000 response-time +samples, and 10,000 error records. Server tests cover invalid thresholds, bad classifier responses, +missing stage-router request IDs, target errors, and retry count 11, which is one above the maximum. ## Run the 48-hour test Choose concurrency from the release capacity plan. Increase it in short -rehearsals until you find the highest expected steady load that remains below +test runs until you find the highest expected steady load that remains below the backend's rate limit. Use that load for the 48-hour run. An overload test that spends most of its time throttled does not measure release stability. @@ -128,7 +127,7 @@ large usage charges against a metered backend. Use a dedicated test deployment, estimate the request volume first with a short run, and get approval for any paid-provider cost. -Use a five-minute rehearsal to confirm the route and result files: +Use a five-minute run to confirm the route and result files: ```bash ./target/release/switchyard-soak \ diff --git a/scripts/soak_rehearsal.toml b/scripts/local_soak_test.toml similarity index 92% rename from scripts/soak_rehearsal.toml rename to scripts/local_soak_test.toml index 5192d7e87..d412916c6 100644 --- a/scripts/soak_rehearsal.toml +++ b/scripts/local_soak_test.toml @@ -1,6 +1,8 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +# Local routes used by run_local_soak_test.py; no provider credentials are required. + schema_version = 1 [llm_clients.mock] diff --git a/scripts/soak_rehearsal.py b/scripts/run_local_soak_test.py similarity index 92% rename from scripts/soak_rehearsal.py rename to scripts/run_local_soak_test.py index 7a2ca63b8..1b662f597 100755 --- a/scripts/soak_rehearsal.py +++ b/scripts/run_local_soak_test.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Rehearse every Switchyard route with embedded VidaiMock, oha, AIPerf, and the soak test.""" +"""Run every Switchyard route with embedded VidaiMock, oha, AIPerf, and the soak test.""" import argparse import json @@ -42,7 +42,7 @@ class Child: @dataclass(frozen=True) class RequiredBinary: - """One command the rehearsal must find before starting any process.""" + """One command the local soak test must find before starting any process.""" label: str env_var: str @@ -78,8 +78,8 @@ def parser() -> argparse.ArgumentParser: """Build the plain-English command-line interface.""" command = argparse.ArgumentParser( description=( - "Start a local VidaiMock-backed Switchyard stack, smoke every route, then run oha, " - "AIPerf, and switchyard-soak." + "Start VidaiMock and a local Switchyard server, send one request through every route, " + "then run oha, AIPerf, and switchyard-soak." ), formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) @@ -219,8 +219,8 @@ def run_checked(name: str, command: Sequence[str], log_path: Path) -> None: raise RuntimeError(f"{name} failed with status {result.returncode}; see {log_path}") -def smoke_routes(base_url: str, output_path: Path) -> None: - """Send one production HTTP request through every configured route.""" +def check_routes(base_url: str, output_path: Path) -> None: + """Send one HTTP request through every configured route.""" ordinary = [{"role": "user", "content": "Reply with exactly OK."}] stage_edge = [ {"role": "user", "content": "Fix the build."}, @@ -273,11 +273,11 @@ def smoke_routes(base_url: str, output_path: Path) -> None: def default_output_dir(repo_root: Path) -> Path: """Choose a new timestamped directory under the repository.""" stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") - return repo_root / "soak-rehearsal-results" / stamp + return repo_root / "local-soak-test-results" / stamp -def run_rehearsal(args: argparse.Namespace) -> Path: - """Run the complete local rehearsal and return its output directory.""" +def run_local_soak_test(args: argparse.Namespace) -> Path: + """Run the complete local soak test and return its output directory.""" repo_root = Path(__file__).resolve().parents[1] rust_build = ( "Build the Rust commands with: cargo build --release -p switchyard-server " @@ -334,7 +334,7 @@ def run_rehearsal(args: argparse.Namespace) -> Path: output_dir = (args.output_dir or default_output_dir(repo_root)).resolve() output_dir.mkdir(parents=True, exist_ok=False) config_path = output_dir / "routes.toml" - shutil.copy2(repo_root / "scripts/soak_rehearsal.toml", config_path) + shutil.copy2(repo_root / "scripts/local_soak_test.toml", config_path) run_checked( "switchyard-server config validation", @@ -366,7 +366,7 @@ def run_rehearsal(args: argparse.Namespace) -> Path: children.append(server) base_url = f"http://127.0.0.1:{args.server_port}" wait_for_health(server, f"{base_url}/health", "ok") - smoke_routes(base_url, output_dir / "route-smoke.jsonl") + check_routes(base_url, output_dir / "route-checks.jsonl") oha_body = output_dir / "oha-request.json" oha_body.write_text( @@ -436,7 +436,7 @@ def run_rehearsal(args: argparse.Namespace) -> Path: ) run_checked( - "switchyard-soak endpoint and streaming matrix", + "switchyard-soak API and streaming requests", [ soak_bin, "--base-url", @@ -466,14 +466,14 @@ def run_rehearsal(args: argparse.Namespace) -> Path: def main(argv: Sequence[str] | None = None) -> int: - """Parse arguments, run the rehearsal, and print one actionable failure.""" + """Parse arguments, run the local soak test, and print one actionable failure.""" args = parser().parse_args(argv) try: - output_dir = run_rehearsal(args) + output_dir = run_local_soak_test(args) except (OSError, RuntimeError) as error: - print(f"soak rehearsal failed: {error}", file=sys.stderr) + print(f"local soak test failed: {error}", file=sys.stderr) return 1 - print(f"Soak rehearsal passed; results: {output_dir}") + print(f"Local soak test passed; results: {output_dir}") return 0 diff --git a/tests/test_soak_rehearsal.py b/tests/test_soak_rehearsal.py deleted file mode 100644 index 8be8b8854..000000000 --- a/tests/test_soak_rehearsal.py +++ /dev/null @@ -1,20 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -import subprocess -import sys -from pathlib import Path - - -def test_help_runs_on_supported_python() -> None: - script = Path(__file__).resolve().parents[1] / "scripts" / "soak_rehearsal.py" - - result = subprocess.run( - [sys.executable, str(script), "--help"], - check=False, - capture_output=True, - text=True, - ) - - assert result.returncode == 0, result.stderr - assert "--mock-latency-ms" in result.stdout From 123a7d2eefd8ea637c5c30d3637c2f757d4c9d60 Mon Sep 17 00:00:00 2001 From: Elyas Mehtabuddin Date: Tue, 18 Aug 2026 09:51:50 -0700 Subject: [PATCH 5/9] fix(operations): validate soak stream completion Signed-off-by: Elyas Mehtabuddin --- crates/libsy/src/core/algorithm.rs | 4 - crates/switchyard-soak/src/client.rs | 177 +++++++++++++++++++++++++-- crates/switchyard-soak/tests/soak.rs | 14 ++- 3 files changed, 177 insertions(+), 18 deletions(-) diff --git a/crates/libsy/src/core/algorithm.rs b/crates/libsy/src/core/algorithm.rs index 13f1cd441..47ddfb668 100644 --- a/crates/libsy/src/core/algorithm.rs +++ b/crates/libsy/src/core/algorithm.rs @@ -422,10 +422,6 @@ mod tests { LibsyError::external("test", TestError(message)) } - /// Build a routed decision for orchestration tests. - fn test_decision(selected_model_id: String) -> Decision { - Decision::new(selected_model_id, None, true) - } /// Trivial algo used only to exercise the orchestrator: calls the first target /// and returns its response as the routing outcome. struct TestAlgo { diff --git a/crates/switchyard-soak/src/client.rs b/crates/switchyard-soak/src/client.rs index e8fd7a7ed..a21475dcc 100644 --- a/crates/switchyard-soak/src/client.rs +++ b/crates/switchyard-soak/src/client.rs @@ -120,6 +120,7 @@ fn parse_metrics(text: &str) -> Metrics { metrics } +#[derive(Debug)] pub struct RequestError { pub kind: String, pub detail: String, @@ -141,6 +142,124 @@ impl RequestError { } } +struct StreamValidator { + endpoint: Endpoint, + event_name: Option, + saw_json: bool, + saw_terminal: bool, +} + +impl StreamValidator { + fn new(endpoint: Endpoint) -> Self { + Self { + endpoint, + event_name: None, + saw_json: false, + saw_terminal: false, + } + } + + fn read_line(&mut self, line: &[u8]) -> Result<(), RequestError> { + let line = line.strip_suffix(b"\r").unwrap_or(line); + if line.is_empty() { + self.event_name = None; + return Ok(()); + } + let line = std::str::from_utf8(line) + .map_err(|error| RequestError::new("invalid_stream", error.to_string()))?; + if let Some(event_name) = line.strip_prefix("event:") { + self.event_name = Some(event_name.trim_start().to_string()); + return Ok(()); + } + let Some(data) = line.strip_prefix("data:") else { + return Ok(()); + }; + let data = data.strip_prefix(' ').unwrap_or(data); + if self.saw_terminal { + return Err(RequestError::new( + "invalid_stream", + "stream contained data after its terminal event", + )); + } + if data == "[DONE]" { + if self.endpoint != Endpoint::Chat { + return Err(RequestError::new( + "invalid_stream", + format!( + "{} stream contained an OpenAI [DONE] marker", + self.endpoint.as_str() + ), + )); + } + self.saw_terminal = true; + return Ok(()); + } + + let payload: Value = serde_json::from_str(data).map_err(|error| { + RequestError::new("invalid_stream", format!("invalid SSE JSON: {error}")) + })?; + let event_type = payload.get("type").and_then(Value::as_str); + if self.event_name.as_deref() == Some("error") + || event_type == Some("error") + || payload.get("error").is_some() + { + let detail = payload + .pointer("/error/message") + .or_else(|| payload.get("message")) + .and_then(Value::as_str) + .unwrap_or("stream returned an error event"); + return Err(RequestError::new("stream_error", truncate(detail, 500))); + } + + match self.endpoint { + Endpoint::Chat => {} + Endpoint::Messages | Endpoint::Responses => { + let event_type = event_type.ok_or_else(|| { + RequestError::new("invalid_stream", "SSE data did not contain an event type") + })?; + if self.event_name.as_deref() != Some(event_type) { + return Err(RequestError::new( + "invalid_stream", + format!( + "SSE event name {:?} did not match data type {event_type:?}", + self.event_name + ), + )); + } + self.saw_terminal = matches!( + (self.endpoint, event_type), + (Endpoint::Messages, "message_stop") + | ( + Endpoint::Responses, + "response.completed" | "response.incomplete" + ) + ); + } + } + self.saw_json = true; + Ok(()) + } + + fn finish(self) -> Result<(), RequestError> { + if !self.saw_json { + return Err(RequestError::new( + "empty_stream", + "successful streaming response contained no JSON events", + )); + } + if !self.saw_terminal { + return Err(RequestError::new( + "invalid_stream", + format!( + "{} stream ended without its terminal event", + self.endpoint.as_str() + ), + )); + } + Ok(()) + } +} + /// Send one request and validate its response body or event stream. pub async fn send_request( client: &Client, @@ -181,25 +300,27 @@ pub async fn send_request( )); } let mut bytes = response.bytes_stream(); - let mut received_data = false; + let mut pending = Vec::new(); + let mut validator = StreamValidator::new(endpoint); while let Some(chunk) = bytes.next().await { match chunk { Ok(chunk) => { - received_data = - received_data || chunk.iter().any(|byte| !byte.is_ascii_whitespace()); + pending.extend_from_slice(&chunk); + while let Some(newline) = pending.iter().position(|byte| *byte == b'\n') { + let mut line = pending.drain(..=newline).collect::>(); + line.pop(); + validator.read_line(&line)?; + } } Err(error) => { return Err(RequestError::transport(&error)); } } } - if !received_data { - return Err(RequestError::new( - "empty_stream", - "successful streaming response contained no data", - )); + if !pending.is_empty() { + validator.read_line(&pending)?; } - return Ok(()); + return validator.finish(); } let status = response.status(); @@ -356,4 +477,42 @@ mod tests { assert_eq!(metrics.requests, Some(42.0)); assert_eq!(metrics.errors, Some(3.0)); } + + #[test] + fn stream_validator_accepts_each_public_terminal_event() -> Result<(), RequestError> { + for (endpoint, stream) in [ + (Endpoint::Chat, "data: {\"choices\":[]}\n\ndata: [DONE]\n\n"), + ( + Endpoint::Messages, + "event: message_stop\ndata: {\"type\":\"message_stop\"}\n\n", + ), + ( + Endpoint::Responses, + "event: response.completed\ndata: {\"type\":\"response.completed\"}\n\n", + ), + ] { + let mut validator = StreamValidator::new(endpoint); + for line in stream.split('\n') { + validator.read_line(line.as_bytes())?; + } + validator.finish()?; + } + Ok(()) + } + + #[test] + fn stream_validator_rejects_errors_and_missing_terminal_events() { + let mut error = StreamValidator::new(Endpoint::Chat); + let result = error.read_line(b"data: {\"error\":{\"message\":\"boom\"}}"); + assert_eq!(result.unwrap_err().kind, "stream_error"); + + let mut incomplete = StreamValidator::new(Endpoint::Responses); + incomplete + .read_line(b"event: response.output_text.delta") + .expect("event name should parse"); + incomplete + .read_line(b"data: {\"type\":\"response.output_text.delta\",\"delta\":\"OK\"}") + .expect("data should parse"); + assert_eq!(incomplete.finish().unwrap_err().kind, "invalid_stream"); + } } diff --git a/crates/switchyard-soak/tests/soak.rs b/crates/switchyard-soak/tests/soak.rs index 8db51aa6e..7a518a61e 100644 --- a/crates/switchyard-soak/tests/soak.rs +++ b/crates/switchyard-soak/tests/soak.rs @@ -99,11 +99,15 @@ fn switchyard(is_chat_broken: bool) -> (Router, Arc>>) { } fn response(body: &Value, field: &str) -> Response { if body.get("stream") == Some(&Value::Bool(true)) { - ( - [(header::CONTENT_TYPE, "text/event-stream")], - "data: {}\n\n", - ) - .into_response() + let stream = match field { + "choices" => "data: {\"choices\":[]}\n\ndata: [DONE]\n\n", + "content" => "event: message_stop\ndata: {\"type\":\"message_stop\"}\n\n", + "output" => { + "event: response.completed\ndata: {\"type\":\"response.completed\"}\n\n" + } + _ => "", + }; + ([(header::CONTENT_TYPE, "text/event-stream")], stream).into_response() } else { let mut response = serde_json::Map::new(); response.insert(field.to_string(), json!([])); From a6aa9691c074444620c2e4d1516ce1e771303376 Mon Sep 17 00:00:00 2001 From: Elyas Mehtabuddin Date: Tue, 18 Aug 2026 11:10:50 -0700 Subject: [PATCH 6/9] feat(operations): add routing performance report Signed-off-by: Elyas Mehtabuddin --- .gitignore | 1 + crates/switchyard-soak/README.md | 9 +- docs/operations/soak_test.md | 58 ++- scripts/benchmark_routing_algorithms.py | 540 +++++++++++++++++++++++ scripts/local_soak_test.toml | 2 - scripts/run_local_soak_test.py | 183 ++------ tests/test_routing_performance_report.py | 69 +++ 7 files changed, 704 insertions(+), 158 deletions(-) create mode 100755 scripts/benchmark_routing_algorithms.py create mode 100644 tests/test_routing_performance_report.py diff --git a/.gitignore b/.gitignore index 03c031b23..c456be335 100644 --- a/.gitignore +++ b/.gitignore @@ -152,6 +152,7 @@ plans/ bench_results*/ soak-results/ local-soak-test-results/ +routing-benchmark-results/ jobs/**/*.* pytest-of-*/ diff --git a/crates/switchyard-soak/README.md b/crates/switchyard-soak/README.md index ec10a7d45..9b595bf0c 100644 --- a/crates/switchyard-soak/README.md +++ b/crates/switchyard-soak/README.md @@ -72,10 +72,17 @@ in short runs. ## Check the local server and load tools `scripts/run_local_soak_test.py` starts an embedded VidaiMock server and `switchyard-server`, sends -one HTTP request through each configured route, then runs `oha`, NVIDIA AIPerf, and +one HTTP request through each configured route, then runs `oha` and NVIDIA AIPerf sequentially for +every routing algorithm. It writes one Markdown, CSV, and JSON performance report before it runs `switchyard-soak`. VidaiMock returns fixed local responses, so this local test needs no provider key and incurs no inference cost. +Run `scripts/benchmark_routing_algorithms.py` by itself against an existing Switchyard server to +compare the same routes with real model output. The command reports TTFT, ITL, request throughput, +and output-token throughput from AIPerf alongside oha's raw HTTP request rate and latency results. +The operations guide explains how to keep the comparison fair and when to use VidaiMock or a real +backend. + The script needs the built Rust programs plus installed `oha` and AIPerf commands. `switchyard-soak` itself does not require those extra programs and can run by itself against a release server. See the [operations guide](../../docs/operations/soak_test.md) for setup, the local diff --git a/docs/operations/soak_test.md b/docs/operations/soak_test.md index 2780ad815..e5ec5bf6e 100644 --- a/docs/operations/soak_test.md +++ b/docs/operations/soak_test.md @@ -50,6 +50,41 @@ release, then run it directly: cargo build --release -p switchyard-soak ``` +## Compare routing algorithm performance + +Use `scripts/benchmark_routing_algorithms.py` to compare routing algorithms under the same load. +The command runs oha and AIPerf sequentially for every model id, then writes `report.md`, +`report.csv`, and `report.json` beside both tools' raw results: + +```bash +python3.12 scripts/benchmark_routing_algorithms.py \ + --base-url http://127.0.0.1:4000 \ + --model noop=switchyard/noop \ + --model passthrough=switchyard/passthrough \ + --model random=switchyard/random \ + --model llm_classifier=switchyard/classifier \ + --model stage_router=switchyard/stage \ + --concurrency 100 \ + --request-count 1000 \ + --backend-label "release model deployment" +``` + +oha sends non-streaming fixed-body requests and reports the raw HTTP request rate and latency +ceiling. AIPerf sends streaming Chat Completions and reports request latency, time to first token +(TTFT), inter-token latency (ITL), request throughput, and output-token throughput. The command +runs them one after the other because simultaneous runs would compete for the same server capacity +and distort both results. + +To isolate routing overhead, configure every route to use the same target deployment and keep the +prompt, input length, output length, concurrency, and request count fixed. Set `--tokenizer` to the +real model tokenizer when exact token counts matter. AIPerf uses a deterministic workload seed so +each route receives the same synthetic inputs. + +This comparison command does not start VidaiMock or Switchyard. Point it at a running Switchyard +server backed by real models to measure end-to-end TTFT and token throughput. Real-model runs cost +tokens and include provider queuing and model variance, so use a dedicated deployment and repeat the +run before treating a small difference as an algorithm effect. + ## Check the local server and load tools The local test embeds [VidaiMock](https://github.com/vidaiUK/VidaiMock) as a Rust library, so it does @@ -66,17 +101,21 @@ Install [oha](https://github.com/hatoo/oha) and ```bash cargo install oha -uv tool install aiperf +uv tool install --python 3.12 aiperf ``` Then run the local test from the repository root: ```bash -python3.12 scripts/run_local_soak_test.py --duration 10s --concurrency 4 +python3.12 scripts/run_local_soak_test.py \ + --duration 10s \ + --concurrency 4 \ + --request-count 100 ``` -`--duration` controls how long the Rust soak tester runs. The route checks, oha, and AIPerf runs are -short and limited by request count. `--help` explains every flag. Set `OHA_BIN`, `AIPERF_BIN`, +`--duration` controls how long the Rust soak tester runs. `--request-count` controls how many +measured requests each load tool sends to each algorithm. `--help` explains every flag. Set +`OHA_BIN`, `AIPERF_BIN`, `SWITCHYARD_SERVER_BIN`, `SWITCHYARD_SOAK_BIN`, or `SWITCHYARD_SOAK_MOCK_BIN` when a command is not on `PATH` or not under `target/release`. @@ -90,14 +129,19 @@ The script gives each tool one job: | Tool | Job in the local test | |---|---| | Embedded VidaiMock | Supplies local OpenAI-compatible model responses with configurable latency and no provider cost. | -| oha | Sends raw concurrent HTTP requests through the random route and writes a status and latency distribution. | +| oha | Sends raw concurrent HTTP requests through every route and writes a status and latency distribution for each algorithm. | | Python route checks | Sends one Chat Completions request through each route. The stage request includes a critical tool failure so the signal scorer selects the capable target. | -| AIPerf | Sends streaming Chat Completions through passthrough routing and records LLM, token, and response-time results. | +| AIPerf | Sends the same deterministic streaming workload through every route and records LLM, token, and response-time results. | +| Combined report | Joins selected oha and AIPerf metrics by algorithm in Markdown, CSV, and JSON. | | `switchyard-soak` | Runs passthrough routing through Chat Completions, Messages, and Responses, with streaming on and off, while checking server health, metrics, process use, and required results. | `scripts/local_soak_test.toml` exercises `noop`, `random`, `passthrough`, `llm_classifier`, and `stage_router`. It uses the accepted maximum retry count (10), a zero-weight random target, and the -upper classifier and stage thresholds (1.0). The config is validated with +upper classifier and stage thresholds (1.0). Classifier affinity is disabled so every measured +request includes the classifier call. VidaiMock's generic classifier reply is intentionally not a +valid routing verdict, so the local classifier row measures the classifier call and fail-open path. +Use a real classifier backend or a purpose-built classifier stub to measure successful +classification. The config is validated with `switchyard-server --dry-run` before either service starts. The runner does not add limits for target count or recent-turn history because the server does not diff --git a/scripts/benchmark_routing_algorithms.py b/scripts/benchmark_routing_algorithms.py new file mode 100755 index 000000000..646e42883 --- /dev/null +++ b/scripts/benchmark_routing_algorithms.py @@ -0,0 +1,540 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Compare Switchyard routing algorithms with oha and NVIDIA AIPerf.""" + +import argparse +import csv +import json +import os +import re +import shutil +import subprocess +import sys +from collections.abc import Sequence +from dataclasses import asdict, dataclass +from datetime import datetime, timezone +from pathlib import Path + + +@dataclass(frozen=True) +class RequiredBinary: + """One command the benchmark must find before it starts.""" + + label: str + env_var: str + value: str + setup: str + + +@dataclass(frozen=True) +class BenchmarkConfig: + """Inputs shared by every algorithm comparison run.""" + + base_url: str + models: tuple[tuple[str, str], ...] + concurrency: int + request_count: int + backend_label: str + output_dir: Path + oha_bin: str + aiperf_bin: str + tokenizer: str = "builtin" + input_sequence_length: int = 32 + output_sequence_length: int = 8 + + +@dataclass(frozen=True) +class BenchmarkResult: + """Selected oha and AIPerf metrics for one routing algorithm.""" + + algorithm: str + model: str + oha_requests_per_second: float + oha_latency_p50_ms: float + oha_latency_p99_ms: float + aiperf_requests_per_second: float + aiperf_request_latency_p50_ms: float + aiperf_ttft_p50_ms: float + aiperf_ttft_p99_ms: float + aiperf_itl_p50_ms: float | None + aiperf_output_tokens_per_second: float + aiperf_output_tokens_per_second_per_user_p50: float | None + + +def positive_int(value: str) -> int: + """Parse a command-line integer that must be greater than zero.""" + parsed = int(value) + if parsed <= 0: + raise argparse.ArgumentTypeError("must be greater than zero") + return parsed + + +def model_spec(value: str) -> tuple[str, str]: + """Parse LABEL=MODEL and reject labels that cannot be directory names.""" + label, separator, model = value.partition("=") + if not separator or not label or not model: + raise argparse.ArgumentTypeError("must use LABEL=MODEL") + if re.fullmatch(r"[A-Za-z0-9_.-]+", label) is None: + raise argparse.ArgumentTypeError( + "LABEL may contain only letters, numbers, periods, underscores, and hyphens" + ) + return label, model + + +def parser() -> argparse.ArgumentParser: + """Build the command-line interface.""" + command = argparse.ArgumentParser( + description=( + "Run sequential oha and AIPerf jobs for Switchyard models, then write one routing " + "algorithm performance report." + ), + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + command.add_argument("--base-url", required=True, help="URL of a running Switchyard server") + command.add_argument( + "--model", + action="append", + required=True, + type=model_spec, + dest="models", + metavar="LABEL=MODEL", + help="algorithm label and exact model id; repeat for every route to compare", + ) + command.add_argument( + "--concurrency", + type=positive_int, + default=4, + help="concurrent requests used by both load tools", + ) + command.add_argument( + "--request-count", + type=positive_int, + default=100, + help="measured requests sent by each tool for each algorithm", + ) + command.add_argument( + "--tokenizer", + default="builtin", + help="AIPerf tokenizer; use the real model tokenizer for provider benchmarks", + ) + command.add_argument( + "--input-sequence-length", + type=positive_int, + default=32, + help="synthetic AIPerf input token count", + ) + command.add_argument( + "--output-sequence-length", + type=positive_int, + default=8, + help="requested AIPerf output token count", + ) + command.add_argument( + "--backend-label", + default="unspecified backend", + help="backend description recorded in the generated report", + ) + command.add_argument( + "--output-dir", + type=Path, + help="new directory for logs, raw tool results, and the combined report", + ) + command.add_argument( + "--oha-bin", + default=os.environ.get("OHA_BIN", "oha"), + help="oha executable path", + ) + command.add_argument( + "--aiperf-bin", + default=os.environ.get("AIPERF_BIN", "aiperf"), + help="AIPerf executable path", + ) + return command + + +def find_binary(value: str) -> str | None: + """Return an executable path, or None when the command cannot run.""" + if os.sep in value: + path = Path(value).expanduser().resolve() + if path.is_file() and os.access(path, os.X_OK): + return str(path) + else: + found = shutil.which(value) + if found is not None: + return found + return None + + +def resolve_binaries(requirements: Sequence[RequiredBinary]) -> dict[str, str]: + """Resolve every command and print all missing-command setup instructions.""" + resolved = {} + missing = [] + for requirement in requirements: + path = find_binary(requirement.value) + if path is None: + missing.append(requirement) + else: + resolved[requirement.label] = path + + for requirement in missing: + print( + f"warning: {requirement.label} executable not found: {requirement.value}", + file=sys.stderr, + ) + print(f" {requirement.setup}", file=sys.stderr) + print( + f" Or set {requirement.env_var}=/path/to/{Path(requirement.value).name}", + file=sys.stderr, + ) + if missing: + raise RuntimeError(f"install or build the {len(missing)} missing command(s), then rerun") + return resolved + + +def run_checked(name: str, command: Sequence[str], log_path: Path) -> None: + """Run one finite tool and keep its output in a named log file.""" + print(f"Running {name}; log: {log_path}") + with log_path.open("w", encoding="utf-8") as log: + result = subprocess.run( + list(command), + stdout=log, + stderr=subprocess.STDOUT, + text=True, + check=False, + ) + if result.returncode != 0: + raise RuntimeError(f"{name} failed with status {result.returncode}; see {log_path}") + + +def read_json_object(path: Path) -> dict[str, object]: + """Read a JSON object and name the invalid result file in any error.""" + try: + document = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + raise RuntimeError(f"could not read JSON result {path}: {error}") from error + if not isinstance(document, dict): + raise RuntimeError(f"expected a JSON object in {path}") + return document + + +def number_at(document: dict[str, object], group: str, field: str, path: Path) -> float: + """Read one required numeric metric from a nested result object.""" + metrics = document.get(group) + value = metrics.get(field) if isinstance(metrics, dict) else None + if not isinstance(value, int | float): + raise RuntimeError(f"missing numeric {group}.{field} in {path}") + return float(value) + + +def optional_number_at(document: dict[str, object], group: str, field: str) -> float | None: + """Read one metric that AIPerf may omit for a one-token response.""" + metrics = document.get(group) + value = metrics.get(field) if isinstance(metrics, dict) else None + return float(value) if isinstance(value, int | float) else None + + +def milliseconds(value: float, unit: object, metric: str, path: Path) -> float: + """Convert an AIPerf latency metric to milliseconds.""" + if unit == "ms": + return value + if unit in {"s", "seconds"}: + return value * 1000 + raise RuntimeError(f"unsupported {metric} unit {unit!r} in {path}") + + +def aiperf_latency(document: dict[str, object], metric: str, field: str, path: Path) -> float: + """Read one required AIPerf latency statistic in milliseconds.""" + value = number_at(document, metric, field, path) + group = document.get(metric) + unit = group.get("unit") if isinstance(group, dict) else None + return milliseconds(value, unit, metric, path) + + +def optional_aiperf_latency( + document: dict[str, object], metric: str, field: str, path: Path +) -> float | None: + """Read one optional AIPerf latency statistic in milliseconds.""" + value = optional_number_at(document, metric, field) + if value is None: + return None + group = document.get(metric) + unit = group.get("unit") if isinstance(group, dict) else None + return milliseconds(value, unit, metric, path) + + +def parse_result(algorithm: str, model: str, oha_path: Path, aiperf_path: Path) -> BenchmarkResult: + """Combine the stable JSON summaries emitted by oha and AIPerf.""" + oha = read_json_object(oha_path) + aiperf = read_json_object(aiperf_path) + success_rate = number_at(oha, "summary", "successRate", oha_path) + if success_rate != 1: + raise RuntimeError(f"oha success rate for {model} was {success_rate:.2%}; see {oha_path}") + error_count = optional_number_at(aiperf, "error_request_count", "avg") + if error_count is not None and error_count != 0: + raise RuntimeError(f"AIPerf recorded {error_count:g} errors for {model}; see {aiperf_path}") + + return BenchmarkResult( + algorithm=algorithm, + model=model, + oha_requests_per_second=number_at(oha, "summary", "requestsPerSec", oha_path), + oha_latency_p50_ms=number_at(oha, "latencyPercentiles", "p50", oha_path) * 1000, + oha_latency_p99_ms=number_at(oha, "latencyPercentiles", "p99", oha_path) * 1000, + aiperf_requests_per_second=number_at(aiperf, "request_throughput", "avg", aiperf_path), + aiperf_request_latency_p50_ms=aiperf_latency(aiperf, "request_latency", "p50", aiperf_path), + aiperf_ttft_p50_ms=aiperf_latency(aiperf, "time_to_first_token", "p50", aiperf_path), + aiperf_ttft_p99_ms=aiperf_latency(aiperf, "time_to_first_token", "p99", aiperf_path), + aiperf_itl_p50_ms=optional_aiperf_latency( + aiperf, "inter_token_latency", "p50", aiperf_path + ), + aiperf_output_tokens_per_second=number_at( + aiperf, "output_token_throughput", "avg", aiperf_path + ), + aiperf_output_tokens_per_second_per_user_p50=optional_number_at( + aiperf, "output_token_throughput_per_user", "p50" + ), + ) + + +def format_metric(value: float | None) -> str: + """Render a report value without implying precision the benchmark lacks.""" + return "n/a" if value is None else f"{value:.2f}" + + +def write_report(config: BenchmarkConfig, results: Sequence[BenchmarkResult]) -> None: + """Write machine-readable and reviewer-readable comparison reports.""" + rows = [asdict(result) for result in results] + payload = { + "generated_at": datetime.now(timezone.utc).isoformat(), + "backend": config.backend_label, + "base_url": config.base_url, + "concurrency": config.concurrency, + "request_count_per_tool_per_algorithm": config.request_count, + "aiperf_tokenizer": config.tokenizer, + "aiperf_input_sequence_length": config.input_sequence_length, + "aiperf_output_sequence_length": config.output_sequence_length, + "results": rows, + } + (config.output_dir / "report.json").write_text( + f"{json.dumps(payload, indent=2)}\n", encoding="utf-8" + ) + + fieldnames = list(rows[0]) + with (config.output_dir / "report.csv").open("w", encoding="utf-8", newline="") as output: + writer = csv.DictWriter(output, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(rows) + + lines = [ + "# Routing algorithm performance", + "", + f"- Backend: {config.backend_label}", + f"- Concurrency: {config.concurrency}", + f"- Measured requests per tool per algorithm: {config.request_count}", + f"- AIPerf workload: streaming Chat Completions, ISL {config.input_sequence_length}, " + f"OSL {config.output_sequence_length}, tokenizer `{config.tokenizer}`", + "- AIPerf warmup: 1 second per algorithm", + "- Run order: all oha and AIPerf jobs ran sequentially", + "", + "| Algorithm | oha req/s | oha p50 ms | oha p99 ms | AIPerf req/s | " + "request p50 ms | TTFT p50 ms | TTFT p99 ms | ITL p50 ms | output tok/s | " + "output tok/s/user p50 |", + "|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|", + ] + for result in results: + lines.append( + f"| {result.algorithm} | {format_metric(result.oha_requests_per_second)} | " + f"{format_metric(result.oha_latency_p50_ms)} | " + f"{format_metric(result.oha_latency_p99_ms)} | " + f"{format_metric(result.aiperf_requests_per_second)} | " + f"{format_metric(result.aiperf_request_latency_p50_ms)} | " + f"{format_metric(result.aiperf_ttft_p50_ms)} | " + f"{format_metric(result.aiperf_ttft_p99_ms)} | " + f"{format_metric(result.aiperf_itl_p50_ms)} | " + f"{format_metric(result.aiperf_output_tokens_per_second)} | " + f"{format_metric(result.aiperf_output_tokens_per_second_per_user_p50)} |" + ) + lines.extend( + ( + "", + "oha uses non-streaming fixed-body requests to measure the HTTP request-rate and " + "full-response latency ceiling. AIPerf uses streaming requests to measure LLM-aware " + "latency and token throughput. Compare algorithms within one tool's columns; do not " + "compare oha values directly with AIPerf values.", + "", + "A VidaiMock run isolates Switchyard routing and protocol overhead. It does not " + "predict production model capacity. Use the same command against routes backed by " + "the same real model deployment to measure end-to-end token performance.", + "", + ) + ) + (config.output_dir / "report.md").write_text("\n".join(lines), encoding="utf-8") + + +def run_oha(config: BenchmarkConfig, algorithm: str, model: str, output_dir: Path) -> Path: + """Run oha's fixed-body HTTP load for one algorithm.""" + request_path = output_dir / f"{algorithm}-request.json" + result_path = output_dir / f"{algorithm}.json" + request_path.write_text( + json.dumps( + { + "model": model, + "messages": [{"role": "user", "content": "Reply with exactly OK."}], + "max_tokens": config.output_sequence_length, + "stream": False, + } + ), + encoding="utf-8", + ) + run_checked( + f"oha {algorithm} load", + [ + config.oha_bin, + "-n", + str(config.request_count), + "-c", + str(config.concurrency), + "--latency-correction", + "--no-tui", + "--method", + "POST", + "-T", + "application/json", + "-D", + str(request_path), + "--output-format", + "json", + "--output", + str(result_path), + f"{config.base_url.rstrip('/')}/v1/chat/completions", + ], + output_dir / f"{algorithm}.log", + ) + return result_path + + +def run_aiperf( + config: BenchmarkConfig, + algorithm: str, + model: str, + output_dir: Path, +) -> Path: + """Run one isolated AIPerf profile for an algorithm.""" + artifact_dir = output_dir / algorithm + run_checked( + f"AIPerf {algorithm} streaming profile", + [ + config.aiperf_bin, + "profile", + "--model", + model, + "--url", + config.base_url, + "--endpoint-type", + "chat", + "--streaming", + "--tokenizer", + config.tokenizer, + "--use-legacy-max-tokens", + "--isl", + str(config.input_sequence_length), + "--osl", + str(config.output_sequence_length), + "--random-seed", + "42", + "--warmup-duration", + "1", + "--concurrency", + str(config.concurrency), + "--request-count", + str(config.request_count), + "--failed-request-threshold", + "0", + "--request-timeout-seconds", + "30", + "--ui", + "none", + "--artifact-dir", + str(artifact_dir), + ], + output_dir / f"{algorithm}.log", + ) + return artifact_dir / "profile_export_aiperf.json" + + +def run_benchmark(config: BenchmarkConfig) -> Path: + """Run both tools for every algorithm and return the report directory.""" + if not config.models: + raise RuntimeError("at least one algorithm model is required") + labels = [label for label, _model in config.models] + if len(labels) != len(set(labels)): + raise RuntimeError("algorithm labels must be unique") + config.output_dir.mkdir(parents=True, exist_ok=False) + oha_dir = config.output_dir / "oha" + aiperf_dir = config.output_dir / "aiperf" + oha_dir.mkdir() + aiperf_dir.mkdir() + + oha_paths = {} + for algorithm, model in config.models: + oha_paths[algorithm] = run_oha(config, algorithm, model, oha_dir) + aiperf_paths = {} + for algorithm, model in config.models: + aiperf_paths[algorithm] = run_aiperf(config, algorithm, model, aiperf_dir) + results = [ + parse_result(algorithm, model, oha_paths[algorithm], aiperf_paths[algorithm]) + for algorithm, model in config.models + ] + + write_report(config, results) + return config.output_dir + + +def default_output_dir() -> Path: + """Choose a new timestamped directory under the repository.""" + repo_root = Path(__file__).resolve().parents[1] + stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + return repo_root / "routing-benchmark-results" / stamp + + +def main(argv: Sequence[str] | None = None) -> int: + """Parse arguments, run the comparison, and print one actionable failure.""" + args = parser().parse_args(argv) + try: + binaries = resolve_binaries( + ( + RequiredBinary( + "oha", "OHA_BIN", args.oha_bin, "Install oha with: cargo install oha" + ), + RequiredBinary( + "AIPerf", + "AIPERF_BIN", + args.aiperf_bin, + "Install AIPerf with: uv tool install --python 3.12 aiperf", + ), + ) + ) + output_dir = run_benchmark( + BenchmarkConfig( + base_url=args.base_url, + models=tuple(args.models), + concurrency=args.concurrency, + request_count=args.request_count, + tokenizer=args.tokenizer, + input_sequence_length=args.input_sequence_length, + output_sequence_length=args.output_sequence_length, + backend_label=args.backend_label, + output_dir=(args.output_dir or default_output_dir()).resolve(), + oha_bin=binaries["oha"], + aiperf_bin=binaries["AIPerf"], + ) + ) + except (OSError, RuntimeError) as error: + print(f"routing algorithm benchmark failed: {error}", file=sys.stderr) + return 1 + print(f"Routing algorithm benchmark passed; report: {output_dir / 'report.md'}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/local_soak_test.toml b/scripts/local_soak_test.toml index d412916c6..967095094 100644 --- a/scripts/local_soak_test.toml +++ b/scripts/local_soak_test.toml @@ -46,8 +46,6 @@ classifier_target = "classifier" strong_target = "strong" weak_target = "weak" base_threshold = 1.0 -session_affinity = true -message_hash_fallback = true max_output_tokens = 1 [routes.stage] diff --git a/scripts/run_local_soak_test.py b/scripts/run_local_soak_test.py index 1b662f597..342105c87 100755 --- a/scripts/run_local_soak_test.py +++ b/scripts/run_local_soak_test.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Run every Switchyard route with embedded VidaiMock, oha, AIPerf, and the soak test.""" +"""Run every Switchyard route with VidaiMock, load tools, and the soak test.""" import argparse import json @@ -18,16 +18,23 @@ from pathlib import Path from typing import TextIO +from benchmark_routing_algorithms import ( + BenchmarkConfig, + RequiredBinary, + positive_int, + resolve_binaries, + run_benchmark, + run_checked, +) + ROUTES = ( - "switchyard/noop", - "switchyard/random", - "switchyard/passthrough", - "switchyard/classifier", - "switchyard/stage", + ("noop", "switchyard/noop"), + ("random", "switchyard/random"), + ("passthrough", "switchyard/passthrough"), + ("llm_classifier", "switchyard/classifier"), + ("stage_router", "switchyard/stage"), ) MOCK_PORT = 8100 -OHA_REQUESTS = 100 -AIPERF_REQUESTS = 20 @dataclass @@ -40,24 +47,6 @@ class Child: log_path: Path -@dataclass(frozen=True) -class RequiredBinary: - """One command the local soak test must find before starting any process.""" - - label: str - env_var: str - value: str - setup: str - - -def positive_int(value: str) -> int: - """Parse a command-line integer that must be greater than zero.""" - parsed = int(value) - if parsed <= 0: - raise argparse.ArgumentTypeError("must be greater than zero") - return parsed - - def nonnegative_int(value: str) -> int: """Parse a command-line integer that may be zero.""" parsed = int(value) @@ -94,6 +83,12 @@ def parser() -> argparse.ArgumentParser: default=4, help="concurrent requests used by oha, AIPerf, and the soak run", ) + command.add_argument( + "--request-count", + type=positive_int, + default=100, + help="measured requests each load tool sends for each routing algorithm", + ) command.add_argument( "--mock-latency-ms", type=nonnegative_int, @@ -114,45 +109,6 @@ def parser() -> argparse.ArgumentParser: return command -def find_binary(value: str) -> str | None: - """Return an executable path, or None when the command cannot run.""" - if os.sep in value: - path = Path(value).expanduser().resolve() - if path.is_file() and os.access(path, os.X_OK): - return str(path) - else: - found = shutil.which(value) - if found is not None: - return found - return None - - -def resolve_binaries(requirements: Sequence[RequiredBinary]) -> dict[str, str]: - """Resolve every command and print all missing-command setup instructions.""" - resolved = {} - missing = [] - for requirement in requirements: - path = find_binary(requirement.value) - if path is None: - missing.append(requirement) - else: - resolved[requirement.label] = path - - for requirement in missing: - print( - f"warning: {requirement.label} executable not found: {requirement.value}", - file=sys.stderr, - ) - print(f" {requirement.setup}", file=sys.stderr) - print( - f" Or set {requirement.env_var}=/path/to/{Path(requirement.value).name}", - file=sys.stderr, - ) - if missing: - raise RuntimeError(f"install or build the {len(missing)} missing command(s), then rerun") - return resolved - - def start_child(name: str, command: Sequence[str], log_path: Path) -> Child: """Start one long-running process and retain its combined log.""" print(f"Starting {name}; log: {log_path}") @@ -204,21 +160,6 @@ def wait_for_health(child: Child, url: str, expected_status: str | None = None) raise RuntimeError(f"{child.name} did not become healthy at {url}; see {child.log_path}") -def run_checked(name: str, command: Sequence[str], log_path: Path) -> None: - """Run one finite tool and keep its output in a named log file.""" - print(f"Running {name}; log: {log_path}") - with log_path.open("w", encoding="utf-8") as log: - result = subprocess.run( - list(command), - stdout=log, - stderr=subprocess.STDOUT, - text=True, - check=False, - ) - if result.returncode != 0: - raise RuntimeError(f"{name} failed with status {result.returncode}; see {log_path}") - - def check_routes(base_url: str, output_path: Path) -> None: """Send one HTTP request through every configured route.""" ordinary = [{"role": "user", "content": "Reply with exactly OK."}] @@ -241,11 +182,11 @@ def check_routes(base_url: str, output_path: Path) -> None: }, ] records = [] - for route in ROUTES: + for algorithm, route in ROUTES: body = json.dumps( { "model": route, - "messages": stage_edge if route == "switchyard/stage" else ordinary, + "messages": stage_edge if algorithm == "stage_router" else ordinary, "max_tokens": 8, "stream": False, } @@ -321,7 +262,7 @@ def run_local_soak_test(args: argparse.Namespace) -> Path: "AIPerf", "AIPERF_BIN", os.environ.get("AIPERF_BIN", "aiperf"), - "Install AIPerf with: uv tool install aiperf", + "Install AIPerf with: uv tool install --python 3.12 aiperf", ), ) ) @@ -368,71 +309,17 @@ def run_local_soak_test(args: argparse.Namespace) -> Path: wait_for_health(server, f"{base_url}/health", "ok") check_routes(base_url, output_dir / "route-checks.jsonl") - oha_body = output_dir / "oha-request.json" - oha_body.write_text( - json.dumps( - { - "model": "switchyard/random", - "messages": [{"role": "user", "content": "Reply with exactly OK."}], - "max_tokens": 8, - "stream": False, - } - ), - encoding="utf-8", - ) - run_checked( - "oha raw HTTP load", - [ - oha_bin, - "-n", - str(OHA_REQUESTS), - "-c", - str(args.concurrency), - "--no-tui", - "--method", - "POST", - "-T", - "application/json", - "-D", - str(oha_body), - "--output-format", - "json", - "--output", - str(output_dir / "oha.json"), - f"{base_url}/v1/chat/completions", - ], - output_dir / "oha.log", - ) - - run_checked( - "AIPerf streaming profile", - [ - aiperf_bin, - "profile", - "--model", - "switchyard/passthrough", - "--url", - base_url, - "--endpoint-type", - "chat", - "--streaming", - "--tokenizer", - "builtin", - "--use-legacy-max-tokens", - "--isl", - "32", - "--osl", - "8", - "--concurrency", - str(args.concurrency), - "--request-count", - str(AIPERF_REQUESTS), - "--request-timeout-seconds", - "30", - "--artifact-dir", - str(output_dir / "aiperf"), - ], - output_dir / "aiperf.log", + run_benchmark( + BenchmarkConfig( + base_url=base_url, + models=ROUTES, + concurrency=args.concurrency, + request_count=args.request_count, + backend_label=f"VidaiMock with {args.mock_latency_ms} ms response latency", + output_dir=output_dir / "routing-benchmark", + oha_bin=oha_bin, + aiperf_bin=aiperf_bin, + ) ) run_checked( diff --git a/tests/test_routing_performance_report.py b/tests/test_routing_performance_report.py new file mode 100644 index 000000000..4260271e7 --- /dev/null +++ b/tests/test_routing_performance_report.py @@ -0,0 +1,69 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import json + +from scripts.benchmark_routing_algorithms import BenchmarkConfig, parse_result, write_report + + +def test_report_combines_oha_and_aiperf_metrics(tmp_path) -> None: + oha_path = tmp_path / "oha.json" + aiperf_path = tmp_path / "profile_export_aiperf.json" + oha_path.write_text( + json.dumps( + { + "summary": {"successRate": 1.0, "requestsPerSec": 120.5}, + "latencyPercentiles": {"p50": 0.012, "p99": 0.045}, + } + ) + ) + aiperf_path.write_text( + json.dumps( + { + "error_request_count": {"unit": "requests", "avg": 0}, + "request_throughput": {"unit": "requests/sec", "avg": 95.5}, + "request_latency": {"unit": "ms", "p50": 40.0}, + "time_to_first_token": {"unit": "ms", "p50": 15.0, "p99": 35.0}, + "output_token_throughput": {"unit": "tokens/sec", "avg": 620.0}, + } + ) + ) + result = parse_result("random", "switchyard/random", oha_path, aiperf_path) + output_dir = tmp_path / "report" + output_dir.mkdir() + config = BenchmarkConfig( + base_url="http://127.0.0.1:4000", + models=(("random", "switchyard/random"),), + concurrency=100, + request_count=1000, + tokenizer="builtin", + input_sequence_length=32, + output_sequence_length=8, + backend_label="test backend", + output_dir=output_dir, + oha_bin="oha", + aiperf_bin="aiperf", + ) + + write_report(config, (result,)) + + report = (output_dir / "report.md").read_text() + assert "| random | 120.50 | 12.00 | 45.00 | 95.50 |" in report + assert "| 40.00 | 15.00 | 35.00 | n/a | 620.00 | n/a |" in report + assert (output_dir / "report.csv").is_file() + assert json.loads((output_dir / "report.json").read_text())["results"] == [ + { + "algorithm": "random", + "model": "switchyard/random", + "oha_requests_per_second": 120.5, + "oha_latency_p50_ms": 12.0, + "oha_latency_p99_ms": 45.0, + "aiperf_requests_per_second": 95.5, + "aiperf_request_latency_p50_ms": 40.0, + "aiperf_ttft_p50_ms": 15.0, + "aiperf_ttft_p99_ms": 35.0, + "aiperf_itl_p50_ms": None, + "aiperf_output_tokens_per_second": 620.0, + "aiperf_output_tokens_per_second_per_user_p50": None, + } + ] From 1d2cdffdc328f2c5f60458b0df7d234efedc4053 Mon Sep 17 00:00:00 2001 From: Elyas Mehtabuddin Date: Tue, 18 Aug 2026 12:47:12 -0700 Subject: [PATCH 7/9] feat(operations): add routing workload scenarios Signed-off-by: Elyas Mehtabuddin --- Cargo.lock | 1009 +---------------- crates/switchyard-soak/Cargo.toml | 1 - crates/switchyard-soak/README.md | 56 +- .../switchyard-soak/examples/mock_server.rs | 182 ++- crates/switchyard-soak/src/client.rs | 9 +- crates/switchyard-soak/src/lib.rs | 116 +- .../src/scenarios/classifier_mix.rs | 42 + .../src/scenarios/client_cancellation.rs | 26 + .../src/scenarios/context_overflow.rs | 26 + .../src/scenarios/decode_heavy.rs | 30 + .../src/scenarios/failure_pressure.rs | 36 + .../src/scenarios/growing_conversation.rs | 34 + .../src/scenarios/large_tool_catalog.rs | 61 + .../src/scenarios/long_context.rs | 36 + .../src/scenarios/mixed_traffic.rs | 41 + crates/switchyard-soak/src/scenarios/mod.rs | 528 +++++++++ .../src/scenarios/prefix_reuse.rs | 37 + .../src/scenarios/short_interactive.rs | 61 + .../src/scenarios/stage_transitions.rs | 53 + .../src/scenarios/tool_call_burst.rs | 65 ++ crates/switchyard-soak/src/stats.rs | 44 +- crates/switchyard-soak/tests/soak.rs | 95 +- docs/operations/soak_test.md | 128 ++- scripts/benchmark_routing_algorithms.py | 946 ++++++++++++---- scripts/run_local_soak_test.py | 46 +- tests/test_routing_performance_report.py | 165 ++- 26 files changed, 2471 insertions(+), 1402 deletions(-) create mode 100644 crates/switchyard-soak/src/scenarios/classifier_mix.rs create mode 100644 crates/switchyard-soak/src/scenarios/client_cancellation.rs create mode 100644 crates/switchyard-soak/src/scenarios/context_overflow.rs create mode 100644 crates/switchyard-soak/src/scenarios/decode_heavy.rs create mode 100644 crates/switchyard-soak/src/scenarios/failure_pressure.rs create mode 100644 crates/switchyard-soak/src/scenarios/growing_conversation.rs create mode 100644 crates/switchyard-soak/src/scenarios/large_tool_catalog.rs create mode 100644 crates/switchyard-soak/src/scenarios/long_context.rs create mode 100644 crates/switchyard-soak/src/scenarios/mixed_traffic.rs create mode 100644 crates/switchyard-soak/src/scenarios/mod.rs create mode 100644 crates/switchyard-soak/src/scenarios/prefix_reuse.rs create mode 100644 crates/switchyard-soak/src/scenarios/short_interactive.rs create mode 100644 crates/switchyard-soak/src/scenarios/stage_transitions.rs create mode 100644 crates/switchyard-soak/src/scenarios/tool_call_burst.rs diff --git a/Cargo.lock b/Cargo.lock index 0368e0ebb..ff7a91185 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -31,15 +31,6 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" -[[package]] -name = "android_system_properties" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" -dependencies = [ - "libc", -] - [[package]] name = "anstream" version = "1.0.0" @@ -105,12 +96,6 @@ dependencies = [ "rustversion", ] -[[package]] -name = "arraydeque" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d902e3d592a523def97af8f317b08ce16b7ab854c1985a0c671e6f15cebc236" - [[package]] name = "assert-json-diff" version = "2.0.2" @@ -289,18 +274,6 @@ name = "bitflags" version = "2.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" -dependencies = [ - "serde_core", -] - -[[package]] -name = "block-buffer" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" -dependencies = [ - "hybrid-array", -] [[package]] name = "borrow-or-share" @@ -308,16 +281,6 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc0b364ead1874514c8c2855ab558056ebfeb775653e7ae45ff72f28f8f3166c" -[[package]] -name = "bstr" -version = "1.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bb31b46c14244e20ee9984b11bf5c992b91fb6939fea616e3512c8baecdbe5f" -dependencies = [ - "memchr", - "serde_core", -] - [[package]] name = "bumpalo" version = "3.20.3" @@ -371,41 +334,6 @@ dependencies = [ "rand_core 0.10.1", ] -[[package]] -name = "chrono" -version = "0.4.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" -dependencies = [ - "iana-time-zone", - "js-sys", - "num-traits", - "wasm-bindgen", - "windows-link", -] - -[[package]] -name = "chrono-tz" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93698b29de5e97ad0ae26447b344c482a7284c737d9ddc5f9e52b74a336671bb" -dependencies = [ - "chrono", - "chrono-tz-build", - "phf", -] - -[[package]] -name = "chrono-tz-build" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c088aee841df9c3041febbb73934cfc39708749bf96dc827e3359cd39ef11b1" -dependencies = [ - "parse-zoneinfo", - "phf", - "phf_codegen", -] - [[package]] name = "clap" version = "4.6.2" @@ -471,61 +399,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "config" -version = "0.15.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b85f248a4de22d204ceabc6299d89d2c70fbd7f09fea53c06c852369652d8139" -dependencies = [ - "async-trait", - "convert_case", - "json5", - "pathdiff", - "ron", - "rust-ini", - "serde-untagged", - "serde_core", - "serde_json", - "toml", - "winnow", - "yaml-rust2", -] - -[[package]] -name = "const-oid" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" - -[[package]] -name = "const-random" -version = "0.1.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" -dependencies = [ - "const-random-macro", -] - -[[package]] -name = "const-random-macro" -version = "0.1.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" -dependencies = [ - "getrandom 0.2.17", - "once_cell", - "tiny-keccak", -] - -[[package]] -name = "convert_case" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec182b0ca2f35d8fc196cf3404988fd8b8c739a4d270ff118a398feb0cbec1ca" -dependencies = [ - "unicode-segmentation", -] - [[package]] name = "core-foundation" version = "0.10.1" @@ -551,64 +424,6 @@ dependencies = [ "libc", ] -[[package]] -name = "crc32fast" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "crossbeam-channel" -version = "0.5.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d85363c37faeca707aef026efa9f3b34d077bce547e48f770770625c6013679e" -dependencies = [ - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-deque" -version = "0.8.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" -dependencies = [ - "crossbeam-epoch", - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-epoch" -version = "0.9.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" -dependencies = [ - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-utils" -version = "0.8.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" - -[[package]] -name = "crunchy" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" - -[[package]] -name = "crypto-common" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" -dependencies = [ - "hybrid-array", -] - [[package]] name = "data-encoding" version = "2.11.1" @@ -633,35 +448,12 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "092966b41edc516079bdf31ec78a2e0588d1d0c08f78b91d8307215928642b2b" -[[package]] -name = "deranged" -version = "0.5.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" - -[[package]] -name = "deunicode" -version = "1.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abd57806937c9cc163efc8ea3910e00a62e2aeb0b8119f1793a978088f8f6b04" - [[package]] name = "diff" version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" -[[package]] -name = "digest" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" -dependencies = [ - "block-buffer", - "const-oid", - "crypto-common", -] - [[package]] name = "displaydoc" version = "0.2.6" @@ -673,15 +465,6 @@ dependencies = [ "syn 2.0.119", ] -[[package]] -name = "dlv-list" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "442039f5147480ba31067cb00ada1adae6892028e40e45fc5de7b7df6dcc1b5f" -dependencies = [ - "const-random", -] - [[package]] name = "dunce" version = "1.0.5" @@ -703,32 +486,12 @@ dependencies = [ "serde", ] -[[package]] -name = "encoding_rs" -version = "0.8.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" -dependencies = [ - "cfg-if", -] - [[package]] name = "equivalent" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" -[[package]] -name = "erased-serde" -version = "0.4.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2add8a07dd6a8d93ff627029c51de145e12686fbc36ecb298ac22e74cf02dec" -dependencies = [ - "serde", - "serde_core", - "typeid", -] - [[package]] name = "errno" version = "0.3.14" @@ -949,36 +712,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "glob" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" - -[[package]] -name = "globset" -version = "0.4.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e47d37d2ae4464254884b60ab7071be2b876a9c35b696bd018ddcc76847309cd" -dependencies = [ - "aho-corasick", - "bstr", - "log", - "regex-automata", - "regex-syntax", -] - -[[package]] -name = "globwalk" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bf760ebf69878d9fd8f110c89703d90ce35095324d1f1edcb595c63945ee757" -dependencies = [ - "bitflags", - "ignore", - "walkdir", -] - [[package]] name = "h2" version = "0.4.15" @@ -998,24 +731,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "hashbrown" -version = "0.14.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" -dependencies = [ - "ahash", -] - -[[package]] -name = "hashbrown" -version = "0.16.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" -dependencies = [ - "foldhash", -] - [[package]] name = "hashbrown" version = "0.17.1" @@ -1027,15 +742,6 @@ dependencies = [ "foldhash", ] -[[package]] -name = "hashlink" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "824e001ac4f3012dd16a264bec811403a67ca9deb6c102fc5049b32c4574b35f" -dependencies = [ - "hashbrown 0.16.1", -] - [[package]] name = "heck" version = "0.5.0" @@ -1093,30 +799,12 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" -[[package]] -name = "humansize" -version = "2.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cb51c9a029ddc91b07a787f1d86b53ccfa49b0e86688c946ebe8d3555685dd7" -dependencies = [ - "libm", -] - [[package]] name = "humantime" version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15" -[[package]] -name = "hybrid-array" -version = "0.4.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "707114b52a152fa7bdb290cd7cd5912d9467273b6d74e21b8d81aca1f8533f6b" -dependencies = [ - "typenum", -] - [[package]] name = "hyper" version = "1.10.1" @@ -1148,9 +836,7 @@ dependencies = [ "http", "hyper", "hyper-util", - "log", "rustls", - "rustls-native-certs", "tokio", "tokio-rustls", "tower-service", @@ -1179,30 +865,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "iana-time-zone" -version = "0.1.65" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" -dependencies = [ - "android_system_properties", - "core-foundation-sys", - "iana-time-zone-haiku", - "js-sys", - "log", - "wasm-bindgen", - "windows-core", -] - -[[package]] -name = "iana-time-zone-haiku" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" -dependencies = [ - "cc", -] - [[package]] name = "icu_collections" version = "2.2.0" @@ -1306,22 +968,6 @@ dependencies = [ "icu_properties", ] -[[package]] -name = "ignore" -version = "0.4.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b17771570a2b94107741a7b033f19132c2eee21d59d21b24d2ced26500bd66e" -dependencies = [ - "crossbeam-deque", - "globset", - "log", - "memchr", - "regex-automata", - "same-file", - "walkdir", - "winapi-util", -] - [[package]] name = "indexmap" version = "2.14.0" @@ -1329,7 +975,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", - "hashbrown 0.17.1", + "hashbrown", ] [[package]] @@ -1429,17 +1075,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "json5" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96b0db21af676c1ce64250b5f40f3ce2cf27e4e47cb91ed91eb6fe9350b430c1" -dependencies = [ - "pest", - "pest_derive", - "serde", -] - [[package]] name = "jsonptr" version = "0.8.1" @@ -1514,21 +1149,6 @@ version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" -[[package]] -name = "libm" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" - -[[package]] -name = "libmimalloc-sys" -version = "0.1.49" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a45a52f43e1c16f667ccfe4dd8c85b7f7c204fd5e3bf46c5b0db9a5c3c0b8e9" -dependencies = [ - "cc", -] - [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -1583,83 +1203,18 @@ version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" -[[package]] -name = "metrics" -version = "0.23.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3045b4193fbdc5b5681f32f11070da9be3609f189a79f3390706d42587f46bb5" -dependencies = [ - "ahash", - "portable-atomic", -] - -[[package]] -name = "metrics-exporter-prometheus" -version = "0.15.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4f0c8427b39666bf970460908b213ec09b3b350f20c0c2eabcbba51704a08e6" -dependencies = [ - "base64", - "http-body-util", - "hyper", - "hyper-rustls", - "hyper-util", - "indexmap", - "ipnet", - "metrics", - "metrics-util", - "quanta", - "thiserror 1.0.69", - "tokio", - "tracing", -] - -[[package]] -name = "metrics-util" -version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4259040465c955f9f2f1a4a8a16dc46726169bca0f88e8fb2dbeced487c3e828" -dependencies = [ - "crossbeam-epoch", - "crossbeam-utils", - "hashbrown 0.14.5", - "metrics", - "num_cpus", - "quanta", - "sketches-ddsketch", -] - [[package]] name = "micromap" version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a86d3146ed3995b5913c414f6664344b9617457320782e64f0bb44afd49d74" -[[package]] -name = "mimalloc" -version = "0.1.52" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d4139bb28d14ad1facf21d5eb8825051b326e172d216b39f6d31df53cc97862" -dependencies = [ - "libmimalloc-sys", -] - [[package]] name = "mime" version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" -[[package]] -name = "mime_guess" -version = "2.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e" -dependencies = [ - "mime", - "unicase", -] - [[package]] name = "mio" version = "1.2.2" @@ -1719,12 +1274,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "num-conv" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" - [[package]] name = "num-integer" version = "0.1.46" @@ -1876,16 +1425,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "ordered-multimap" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49203cdcae0030493bad186b28da2fa25645fa276a51b6fec8010d281e02ef79" -dependencies = [ - "dlv-list", - "hashbrown 0.14.5", -] - [[package]] name = "outref" version = "0.5.2" @@ -1915,107 +1454,12 @@ dependencies = [ "windows-link", ] -[[package]] -name = "parse-zoneinfo" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f2a05b18d44e2957b88f96ba460715e295bc1d7510468a2f3d3b44535d26c24" -dependencies = [ - "regex", -] - -[[package]] -name = "pathdiff" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" - [[package]] name = "percent-encoding" version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" -[[package]] -name = "pest" -version = "2.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a07a60cc7a4d00c91f95c685609d1d2f79050e6804b70ebedd7650f0b839bcf" -dependencies = [ - "memchr", - "ucd-trie", -] - -[[package]] -name = "pest_derive" -version = "2.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3a83744a5c8455b8b3e0dc5031362780a347c878bdd11584d1a8984228cc88d" -dependencies = [ - "pest", - "pest_generator", -] - -[[package]] -name = "pest_generator" -version = "2.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0cd3451aa3de60d4b9a1e736885e4dea6b31617598026f12256ad566d63304a" -dependencies = [ - "pest", - "pest_meta", - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "pest_meta" -version = "2.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e04d3a0849e241d7dfce834c83b1c5edc8622009e8dd51a12ba1927c32f05496" -dependencies = [ - "pest", -] - -[[package]] -name = "phf" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" -dependencies = [ - "phf_shared", -] - -[[package]] -name = "phf_codegen" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a" -dependencies = [ - "phf_generator", - "phf_shared", -] - -[[package]] -name = "phf_generator" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" -dependencies = [ - "phf_shared", - "rand 0.8.7", -] - -[[package]] -name = "phf_shared" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" -dependencies = [ - "siphasher", -] - [[package]] name = "pin-project-lite" version = "0.2.17" @@ -2043,12 +1487,6 @@ dependencies = [ "zerovec", ] -[[package]] -name = "powerfmt" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" - [[package]] name = "ppv-lite86" version = "0.2.21" @@ -2217,21 +1655,6 @@ dependencies = [ "serde", ] -[[package]] -name = "quanta" -version = "0.12.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3ab5a9d756f0d97bdc89019bd2e4ea098cf9cde50ee7564dde6b81ccc8f06c7" -dependencies = [ - "crossbeam-utils", - "libc", - "once_cell", - "raw-cpuid", - "wasi", - "web-sys", - "winapi", -] - [[package]] name = "quinn" version = "0.11.11" @@ -2308,18 +1731,7 @@ checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" name = "r-efi" version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" - -[[package]] -name = "rand" -version = "0.8.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" -dependencies = [ - "libc", - "rand_chacha 0.3.1", - "rand_core 0.6.4", -] +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rand" @@ -2327,7 +1739,7 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" dependencies = [ - "rand_chacha 0.9.0", + "rand_chacha", "rand_core 0.9.5", ] @@ -2342,16 +1754,6 @@ dependencies = [ "rand_core 0.10.1", ] -[[package]] -name = "rand_chacha" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" -dependencies = [ - "ppv-lite86", - "rand_core 0.6.4", -] - [[package]] name = "rand_chacha" version = "0.9.0" @@ -2362,15 +1764,6 @@ dependencies = [ "rand_core 0.9.5", ] -[[package]] -name = "rand_core" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" -dependencies = [ - "getrandom 0.2.17", -] - [[package]] name = "rand_core" version = "0.9.5" @@ -2395,15 +1788,6 @@ dependencies = [ "rand_core 0.10.1", ] -[[package]] -name = "raw-cpuid" -version = "11.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" -dependencies = [ - "bitflags", -] - [[package]] name = "redox_syscall" version = "0.5.18" @@ -2442,7 +1826,7 @@ dependencies = [ "ahash", "fluent-uri", "getrandom 0.3.4", - "hashbrown 0.17.1", + "hashbrown", "itoa", "micromap", "parking_lot", @@ -2534,65 +1918,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "ron" -version = "0.12.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81116b9531d61eabc41aeb228e4b6b2435bcca3233b98cf3b3077d4e6e9debb3" -dependencies = [ - "bitflags", - "once_cell", - "serde", - "serde_derive", - "typeid", - "unicode-ident", -] - -[[package]] -name = "rust-embed" -version = "8.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9e7760e252aaba7b09f4be00e36476cf585bdb68a53552ac954cdf504ab4bc9" -dependencies = [ - "rust-embed-impl", - "rust-embed-utils", - "walkdir", -] - -[[package]] -name = "rust-embed-impl" -version = "8.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3bcfc4d6f53af43755f7a723e4b6b8794fcce052a178dd8c6c1dadc5f5343097" -dependencies = [ - "mime_guess", - "proc-macro2", - "quote", - "rust-embed-utils", - "syn 2.0.119", - "walkdir", -] - -[[package]] -name = "rust-embed-utils" -version = "8.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42ffa149f6aa81b58a5b3011d01a857c4ed12c7a732d2c51947a4c7c692185f0" -dependencies = [ - "sha2", - "walkdir", -] - -[[package]] -name = "rust-ini" -version = "0.21.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "796e8d2b6696392a43bea58116b667fb4c29727dc5abd27d6acf338bb4f688c7" -dependencies = [ - "cfg-if", - "ordered-multimap", -] - [[package]] name = "rustc-hash" version = "2.1.3" @@ -2628,7 +1953,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c54fcab019b409d04215d3a17cb438fd7fbf192ee61461f20f4fe18704bc138" dependencies = [ "aws-lc-rs", - "log", "once_cell", "rustls-pki-types", "rustls-webpki", @@ -2772,18 +2096,6 @@ dependencies = [ "serde_derive", ] -[[package]] -name = "serde-untagged" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9faf48a4a2d2693be24c6289dbe26552776eb7737074e6722891fadbe6c5058" -dependencies = [ - "erased-serde", - "serde", - "serde_core", - "typeid", -] - [[package]] name = "serde_core" version = "1.0.228" @@ -2849,30 +2161,6 @@ dependencies = [ "serde", ] -[[package]] -name = "serde_yaml" -version = "0.9.34+deprecated" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" -dependencies = [ - "indexmap", - "itoa", - "ryu", - "serde", - "unsafe-libyaml", -] - -[[package]] -name = "sha2" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" -dependencies = [ - "cfg-if", - "cpufeatures", - "digest", -] - [[package]] name = "sharded-slab" version = "0.1.7" @@ -2914,34 +2202,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" -[[package]] -name = "siphasher" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" - -[[package]] -name = "sketches-ddsketch" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85636c14b73d81f541e525f585c0a2109e6744e1565b5c1668e31c70c10ed65c" - [[package]] name = "slab" version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" -[[package]] -name = "slug" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "882a80f72ee45de3cc9a5afeb2da0331d58df69e4e7d8eeb5d3c7784ae67e724" -dependencies = [ - "deunicode", - "wasm-bindgen", -] - [[package]] name = "smallvec" version = "1.15.2" @@ -3137,7 +2403,6 @@ dependencies = [ "serde_json", "tempfile", "tokio", - "vidaimock", ] [[package]] @@ -3153,12 +2418,6 @@ dependencies = [ "thiserror 2.0.18", ] -[[package]] -name = "symlink" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7973cce6668464ea31f176d85b13c7ab3bba2cb3b77a2ed26abd7801688010a" - [[package]] name = "syn" version = "2.0.119" @@ -3220,28 +2479,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "tera" -version = "1.20.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8004bca281f2d32df3bacd59bc67b312cb4c70cea46cbd79dbe8ac5ed206722" -dependencies = [ - "chrono", - "chrono-tz", - "globwalk", - "humansize", - "lazy_static", - "percent-encoding", - "pest", - "pest_derive", - "rand 0.8.7", - "regex", - "serde", - "serde_json", - "slug", - "unicode-segmentation", -] - [[package]] name = "thiserror" version = "1.0.69" @@ -3291,45 +2528,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "time" -version = "0.3.55" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" -dependencies = [ - "deranged", - "num-conv", - "powerfmt", - "serde_core", - "time-core", - "time-macros", -] - -[[package]] -name = "time-core" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" - -[[package]] -name = "time-macros" -version = "0.2.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85" -dependencies = [ - "num-conv", - "time-core", -] - -[[package]] -name = "tiny-keccak" -version = "2.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" -dependencies = [ - "crunchy", -] - [[package]] name = "tinystr" version = "0.8.3" @@ -3487,7 +2685,6 @@ dependencies = [ "tower", "tower-layer", "tower-service", - "tracing", "url", ] @@ -3515,19 +2712,6 @@ dependencies = [ "tracing-core", ] -[[package]] -name = "tracing-appender" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "050686193eb999b4bb3bc2acfa891a13da00f79734704c4b8b4ef1a10b368a3c" -dependencies = [ - "crossbeam-channel", - "symlink", - "thiserror 2.0.18", - "time", - "tracing-subscriber", -] - [[package]] name = "tracing-attributes" version = "0.1.31" @@ -3576,16 +2760,6 @@ dependencies = [ "web-time", ] -[[package]] -name = "tracing-serde" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "704b1aeb7be0d0a84fc9828cae51dab5970fee5088f83d1dd7ee6f6246fc6ff1" -dependencies = [ - "serde", - "tracing-core", -] - [[package]] name = "tracing-subscriber" version = "0.3.23" @@ -3596,15 +2770,12 @@ dependencies = [ "nu-ansi-term", "once_cell", "regex-automata", - "serde", - "serde_json", "sharded-slab", "smallvec", "thread_local", "tracing", "tracing-core", "tracing-log", - "tracing-serde", ] [[package]] @@ -3613,30 +2784,6 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" -[[package]] -name = "typeid" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c" - -[[package]] -name = "typenum" -version = "1.20.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" - -[[package]] -name = "ucd-trie" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" - -[[package]] -name = "unicase" -version = "2.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" - [[package]] name = "unicode-general-category" version = "1.1.0" @@ -3649,18 +2796,6 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" -[[package]] -name = "unicode-segmentation" -version = "1.13.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" - -[[package]] -name = "unsafe-libyaml" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" - [[package]] name = "untrusted" version = "0.9.0" @@ -3691,17 +2826,6 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" -[[package]] -name = "uuid" -version = "1.24.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cefc03fd367c0c6d4305de1b312cf00248c4114f4a0418ce6a6af769e3b0bd9" -dependencies = [ - "getrandom 0.4.3", - "js-sys", - "wasm-bindgen", -] - [[package]] name = "uuid-simd" version = "0.8.0" @@ -3724,42 +2848,6 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" -[[package]] -name = "vidaimock" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8887e1668af80f23c5ee7b774f75e76439d5f9d91ba80784bac6b78bb73d3fe8" -dependencies = [ - "axum", - "base64", - "chrono", - "clap", - "config", - "crc32fast", - "futures", - "glob", - "metrics", - "metrics-exporter-prometheus", - "mimalloc", - "num_cpus", - "once_cell", - "rand 0.9.5", - "regex", - "rust-embed", - "serde", - "serde_json", - "serde_yaml", - "tera", - "tokio", - "tower", - "tower-http", - "tracing", - "tracing-appender", - "tracing-subscriber", - "uuid", - "walkdir", -] - [[package]] name = "vsimd" version = "0.8.0" @@ -3897,22 +2985,6 @@ dependencies = [ "rustls-pki-types", ] -[[package]] -name = "winapi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" -dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", -] - -[[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" - [[package]] name = "winapi-util" version = "0.1.11" @@ -3922,71 +2994,12 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" - -[[package]] -name = "windows-core" -version = "0.62.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" -dependencies = [ - "windows-implement", - "windows-interface", - "windows-link", - "windows-result", - "windows-strings", -] - -[[package]] -name = "windows-implement" -version = "0.60.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "windows-interface" -version = "0.59.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - [[package]] name = "windows-link" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" -[[package]] -name = "windows-result" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" -dependencies = [ - "windows-link", -] - -[[package]] -name = "windows-strings" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" -dependencies = [ - "windows-link", -] - [[package]] name = "windows-sys" version = "0.52.0" @@ -4074,9 +3087,6 @@ name = "winnow" version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" -dependencies = [ - "memchr", -] [[package]] name = "wiremock" @@ -4113,17 +3123,6 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" -[[package]] -name = "yaml-rust2" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "631a50d867fafb7093e709d75aaee9e0e0d5deb934021fcea25ac2fe09edc51e" -dependencies = [ - "arraydeque", - "encoding_rs", - "hashlink", -] - [[package]] name = "yansi" version = "1.0.1" diff --git a/crates/switchyard-soak/Cargo.toml b/crates/switchyard-soak/Cargo.toml index c2b449c32..242dbed17 100644 --- a/crates/switchyard-soak/Cargo.toml +++ b/crates/switchyard-soak/Cargo.toml @@ -35,4 +35,3 @@ tokio.workspace = true axum = "0.8" tempfile = "3" tokio.workspace = true -vidaimock = "0.3" diff --git a/crates/switchyard-soak/README.md b/crates/switchyard-soak/README.md index 9b595bf0c..d6be78844 100644 --- a/crates/switchyard-soak/README.md +++ b/crates/switchyard-soak/README.md @@ -1,15 +1,17 @@ # switchyard-soak `switchyard-soak` keeps a fixed number of requests in flight against one route on a live -`switchyard-server`. It exercises Chat Completions, Messages, and Responses in both streaming and -non-streaming form. It also samples health, metrics, and an optional local server process. The -command writes evidence to a new results directory and exits with status 1 when a release gate -fails. +`switchyard-server`. Its scenario catalog covers short and long inputs, long outputs, shared +prefixes, mixed traffic, growing conversations, tool schemas, tool-call bursts, routing signals, +and bounded failure cases. The short baseline also exercises Chat Completions, Messages, and +Responses in streaming and non-streaming form. The command samples health, metrics, and an +optional local server process, then exits with status 1 when a release gate fails. Build the server and soak tester from the commit being tested: ```bash -cargo build --release -p switchyard-server -p switchyard-soak +cargo build --release -p switchyard-server -p switchyard-soak \ + --bins --example switchyard-soak-mock ``` Run a five-minute check against a route advertised by `GET /v1/models`: @@ -52,8 +54,12 @@ target/release/switchyard-soak \ | `--model ID` | required | Selects one exact route id returned by `GET /v1/models`. The test stops before sending load if the id is missing. | | `--duration TIME` | `48h` | Keeps the load running for this long. Use seconds, minutes, or hours, such as `30s`, `5m`, or `48h`. | | `--concurrency N` | `16` | Keeps this many requests in flight. Each worker waits for its response before sending another request. | -| `--max-output-tokens N` | `32` | Sends this output-token limit with every request. It must be at least 1. | -| `--prompt-bytes N` | `1024` | Adds this many bytes of repeated prefix to each of four prompts. Larger values put more pressure on request memory and prefix caching. | +| `--max-output-tokens N` | `32` | Sets the default output limit. `decode-heavy` uses its bounded 512-token and 1,024-token limits. | +| `--prompt-bytes N` | `1024` | Sizes the repeated input used by short and prefix-reuse scenarios. Larger values put more pressure on request memory and prefix caching. | +| `--scenario-set NAME` | `standard` | Selects `core`, `agentic`, `resilience`, `standard`, or `all`. `standard` includes core and agentic scenarios without expected failures. | +| `--scenario ID` | none | Selects one exact scenario id. Repeat the flag to select more scenarios; explicit ids take precedence over `--scenario-set`. | +| `--context-window-tokens N` | `32768` | Bounds generated long-context and overflow inputs. The command rejects values above 1,000,000. | +| `--export-scenarios PATH` | none | Writes a manifest and AIPerf `inputs-json` files, then exits without contacting a server. The performance script uses this mode. | | `--request-timeout SECONDS` | `120` | Limits connection time and idle time between response bytes. A healthy stream may run longer if it keeps producing data. | | `--report-interval SECONDS` | `60` | Sets how often the test samples health, metrics, RSS, CPU, and interval latency. | | `--invalid-canary-interval SECONDS` | `300` | Sends malformed input this often, expects HTTP 400, and then checks recovery. Set it to `0` only when a permissive mock cannot reject the canary. | @@ -65,23 +71,35 @@ target/release/switchyard-soak \ | `--help` | n/a | Prints the command reference and examples. | | `--version` | n/a | Prints the crate version. | -The inference workers use a fixed six-case cycle: three public endpoint formats multiplied by -streaming on and off. A deterministic cycle makes missing endpoint or streaming coverage visible -in short runs. +The scenario source lives in `src/scenarios/`. Each request shape has one file and a pure builder. +The exported manifest is the only scenario input read by the Python performance runner, so model +ids, sessions, tools, and message histories cannot drift between the Rust soak and AIPerf runs. +Concurrency knees and traffic bursts are load profiles attached to the short baseline, not +duplicate request scenarios. + +The catalog contains: + +| Group | Scenarios | +|---|---| +| Core | `short-interactive`, `long-context`, `decode-heavy`, `prefix-reuse`, `mixed-traffic` | +| Agentic | `growing-conversation`, `large-tool-catalog`, `tool-call-burst`, `stage-transitions`, `classifier-mix` | +| Resilience | `context-overflow`, `failure-pressure`, `client-cancellation` | ## Check the local server and load tools -`scripts/run_local_soak_test.py` starts an embedded VidaiMock server and `switchyard-server`, sends -one HTTP request through each configured route, then runs `oha` and NVIDIA AIPerf sequentially for -every routing algorithm. It writes one Markdown, CSV, and JSON performance report before it runs -`switchyard-soak`. VidaiMock returns fixed local responses, so this local test needs no provider key -and incurs no inference cost. +`scripts/run_local_soak_test.py` starts the crate's request-aware scenario backend and +`switchyard-server`, sends one HTTP request through each configured route, then runs `oha` and +NVIDIA AIPerf sequentially for every routing algorithm. The backend returns valid classifier +verdicts and injects the context, retry, stream, and cancellation cases named by resilience +scenarios. It resets transient-failure counters before each AIPerf cell so every routing algorithm +receives the same attempts. The local test needs no provider key and incurs no inference cost. Run `scripts/benchmark_routing_algorithms.py` by itself against an existing Switchyard server to -compare the same routes with real model output. The command reports TTFT, ITL, request throughput, -and output-token throughput from AIPerf alongside oha's raw HTTP request rate and latency results. -The operations guide explains how to keep the comparison fair and when to use VidaiMock or a real -backend. +compare the same routes with real model output. oha runs only for the fixed `short-interactive` +baseline. AIPerf replays every selected session and reports TTFT, ITL, request throughput, output +token throughput, multi-run confidence, selected-target calls, classifier calls, and routing +overhead. The operations guide explains how to keep the comparison fair and when to use the local +backend or real models. The script needs the built Rust programs plus installed `oha` and AIPerf commands. `switchyard-soak` itself does not require those extra programs and can run by itself against a diff --git a/crates/switchyard-soak/examples/mock_server.rs b/crates/switchyard-soak/examples/mock_server.rs index b4231302c..50a7ffaa3 100644 --- a/crates/switchyard-soak/examples/mock_server.rs +++ b/crates/switchyard-soak/examples/mock_server.rs @@ -1,49 +1,197 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Local VidaiMock server used by `scripts/run_local_soak_test.py`. +//! Request-aware local backend used by `scripts/run_local_soak_test.py`. +use std::collections::HashMap; use std::process::ExitCode; +use std::sync::Arc; +use std::time::Duration; +use axum::Router; +use axum::extract::{Json, State}; +use axum::http::{StatusCode, header}; +use axum::response::{IntoResponse, Response}; +use axum::routing::{get, post}; use clap::Parser; -use vidaimock::MockServer; +use parking_lot::Mutex; +use serde_json::{Value, json}; -/// Start the embedded mock backend used by the local soak test. #[derive(Parser)] #[command( name = "switchyard-soak-mock", - about = "Start VidaiMock for the local Switchyard soak test", + about = "Start the request-aware backend for the local Switchyard scenario test", after_long_help = "Example:\n cargo run --release -p switchyard-soak --example switchyard-soak-mock -- --port 8100 --latency-ms 40", version )] struct Args { - /// Local TCP port used by the mock backend. + /// Local TCP port used by the backend. #[arg(long, default_value_t = 8100)] port: u16, - /// Artificial delay, in milliseconds, added to every mock response. + /// Artificial delay, in milliseconds, added to ordinary responses. #[arg(long, default_value_t = 40)] latency_ms: u64, } +#[derive(Clone)] +struct BackendState { + latency: Duration, + attempts: Arc>>, +} + +fn scenario_marker(body: &Value) -> Option<&str> { + body.get("messages")? + .as_array()? + .iter() + .filter_map(|message| message.get("content").and_then(Value::as_str)) + .find_map(|content| { + let marker = content.split_once("[scenario:")?.1; + marker.split_once(']').map(|(name, _rest)| name) + }) +} + +fn classifier_content(marker: Option<&str>) -> &'static str { + if marker == Some("classifier_invalid") { + return "not a JSON verdict"; + } + if marker == Some("classifier_hard") { + return r#"{"crux":"distributed diagnosis","primary_rule":"LIM-1","capability_boundary":"unsupported","p_solve":0.1}"#; + } + r#"{"crux":"bounded task","primary_rule":"SUP-1","capability_boundary":"supported","p_solve":1.0}"# +} + +fn completion(model: &str, content: &str) -> Value { + json!({ + "id": "chatcmpl-switchyard-soak", + "object": "chat.completion", + "model": model, + "choices": [{ + "index": 0, + "message": {"role": "assistant", "content": content}, + "finish_reason": "stop" + }], + "usage": {"prompt_tokens": 32, "completion_tokens": 2, "total_tokens": 34} + }) +} + +fn stream(model: &str, truncated: bool) -> Response { + let first = json!({ + "id": "chatcmpl-switchyard-soak", + "object": "chat.completion.chunk", + "model": model, + "choices": [{ + "index": 0, + "delta": {"role": "assistant", "content": "OK"}, + "finish_reason": null + }] + }); + let end = json!({ + "id": "chatcmpl-switchyard-soak", + "object": "chat.completion.chunk", + "model": model, + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 32, "completion_tokens": 2, "total_tokens": 34} + }); + let body = if truncated { + format!("data: {first}\n\n") + } else { + format!("data: {first}\n\ndata: {end}\n\ndata: [DONE]\n\n") + }; + ([(header::CONTENT_TYPE, "text/event-stream")], body).into_response() +} + +async fn chat(State(state): State, Json(body): Json) -> Response { + let model = body + .get("model") + .and_then(Value::as_str) + .unwrap_or_default(); + let marker = scenario_marker(&body); + if marker == Some("client_cancellation") { + tokio::time::sleep(Duration::from_secs(2)).await; + } else { + tokio::time::sleep(state.latency).await; + } + + if model == "mock/classifier" { + return Json(completion(model, classifier_content(marker))).into_response(); + } + if model == "mock/weak" && marker == Some("context_overflow") { + return ( + StatusCode::BAD_REQUEST, + Json(json!({ + "error": { + "code": "context_length_exceeded", + "message": "the weak target context window is too small" + } + })), + ) + .into_response(); + } + + for (name, status) in [ + ("upstream_429", StatusCode::TOO_MANY_REQUESTS), + ("upstream_500", StatusCode::INTERNAL_SERVER_ERROR), + ] { + if marker == Some(name) { + let key = format!("{model}:{name}"); + let attempt = { + let mut attempts = state.attempts.lock(); + let attempt = attempts.entry(key).or_default(); + *attempt += 1; + *attempt + }; + if attempt <= 2 { + return ( + status, + Json( + json!({"error": {"message": format!("injected {name} attempt {attempt}")}}), + ), + ) + .into_response(); + } + } + } + + if body.get("stream") == Some(&Value::Bool(true)) { + return stream(model, marker == Some("truncated_stream")); + } + Json(completion(model, "OK")).into_response() +} + async fn run(args: Args) -> Result<(), String> { if args.port == 0 { return Err("--port must be greater than zero".to_string()); } - - let server = MockServer::builder() - .bind(format!("127.0.0.1:{}", args.port)) - .mode("realistic") - .latency_ms(args.latency_ms) - .start() + let listener = tokio::net::TcpListener::bind(("127.0.0.1", args.port)) .await .map_err(|error| error.to_string())?; - - println!("VidaiMock is ready at {}", server.base_url()); - tokio::signal::ctrl_c() + let address = listener.local_addr().map_err(|error| error.to_string())?; + let state = BackendState { + latency: Duration::from_millis(args.latency_ms), + attempts: Arc::new(Mutex::new(HashMap::new())), + }; + let app = Router::new() + .route("/health", get(|| async { Json(json!({"status": "ok"})) })) + .route( + "/reset", + post(|State(state): State| async move { + state.attempts.lock().clear(); + Json(json!({"status": "reset"})) + }), + ) + .route("/v1/chat/completions", post(chat)) + .with_state(state); + println!("Scenario backend is ready at http://{address}"); + axum::serve(listener, app) + .with_graceful_shutdown(async { + if let Err(error) = tokio::signal::ctrl_c().await { + eprintln!("could not register the shutdown signal: {error}"); + std::future::pending::<()>().await; + } + }) .await - .map_err(|error| format!("could not wait for shutdown: {error}"))?; - server.shutdown().await.map_err(|error| error.to_string()) + .map_err(|error| error.to_string()) } #[tokio::main] diff --git a/crates/switchyard-soak/src/client.rs b/crates/switchyard-soak/src/client.rs index a21475dcc..5fe7074b1 100644 --- a/crates/switchyard-soak/src/client.rs +++ b/crates/switchyard-soak/src/client.rs @@ -265,11 +265,18 @@ pub async fn send_request( client: &Client, base_url: &str, endpoint: Endpoint, + session_id: &str, body: &Value, ) -> Result<(), RequestError> { let url = format!("{base_url}{}", endpoint.path()); let stream = body.get("stream").and_then(Value::as_bool).unwrap_or(false); - let response = match client.post(&url).json(body).send().await { + let response = match client + .post(&url) + .header("x-switchyard-session-id", session_id) + .json(body) + .send() + .await + { Ok(response) => response, Err(error) => return Err(RequestError::transport(&error)), }; diff --git a/crates/switchyard-soak/src/lib.rs b/crates/switchyard-soak/src/lib.rs index d194cd19e..779b70093 100644 --- a/crates/switchyard-soak/src/lib.rs +++ b/crates/switchyard-soak/src/lib.rs @@ -5,6 +5,7 @@ mod client; mod report; +mod scenarios; mod stats; use std::fs; @@ -12,7 +13,7 @@ use std::future::Future; use std::path::{Path, PathBuf}; use std::process::ExitCode; use std::sync::Arc; -use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::atomic::{AtomicBool, Ordering}; use std::time::{Duration, Instant}; use clap::Parser; @@ -22,18 +23,17 @@ use reqwest::header::{AUTHORIZATION, HeaderMap, HeaderValue}; use serde_json::json; use tokio::sync::Notify; -use crate::client::{Endpoint, preflight, request_body, send_request}; +use crate::client::{Endpoint, preflight, send_request}; use crate::report::{ResultsWriter, invalid_request_canary, reporter}; -use crate::stats::{ - RunStats, build_prompt_pool, build_summary, now_utc_string, round3, utc_dir_stamp, -}; +use crate::scenarios::{Scenario, ScenarioOptions, ScenarioSet}; +use crate::stats::{RunStats, build_summary, now_utc_string, round3, utc_dir_stamp}; /// Command-line arguments for the soak test. #[derive(Parser)] #[command( name = "switchyard-soak", about = "Run a sustained, closed-loop load test against a live Switchyard server", - after_long_help = "Examples:\n switchyard-soak --model switchyard/general --duration 5m --concurrency 4\n switchyard-soak --model switchyard/general --duration 48h --server-pid 1234 --max-rss-growth-mib 512\n\nThis command does not require VidaiMock, oha, or AIPerf. The optional scripts/run_local_soak_test.py command uses an embedded VidaiMock helper and warns before it starts when oha or AIPerf is missing.", + after_long_help = "Examples:\n switchyard-soak --model switchyard/general --duration 5m --concurrency 4\n switchyard-soak --model switchyard/general --scenario tool-call-burst --duration 5m\n switchyard-soak --model switchyard/general --export-scenarios scenarios\n\nThis command does not require oha or AIPerf. scripts/benchmark_routing_algorithms.py uses the exported scenario files with both tools.", version )] pub struct Args { @@ -53,7 +53,7 @@ pub struct Args { #[arg(long, default_value_t = 16)] concurrency: usize, - /// Output-token limit sent with every inference request. + /// Default output-token limit; decode-heavy uses bounded 512 and 1,024 limits. #[arg(long, default_value_t = 32)] max_output_tokens: u32, @@ -61,6 +61,22 @@ pub struct Args { #[arg(long, default_value_t = 1024)] prompt_bytes: usize, + /// Named scenario collection; explicit --scenario values take precedence. + #[arg(long, value_enum, default_value_t = ScenarioSet::Standard)] + scenario_set: ScenarioSet, + + /// Exact scenario id to run; repeat to select more than one scenario. + #[arg(long, action = clap::ArgAction::Append)] + scenario: Vec, + + /// Context-window size used to bound generated long-context scenarios. + #[arg(long, default_value_t = 32_768)] + context_window_tokens: usize, + + /// Export selected AIPerf input files and exit without contacting a server. + #[arg(long)] + export_scenarios: Option, + /// Seconds allowed to connect or wait between response bytes; an active stream may run longer. #[arg(long, default_value_t = 120.0)] request_timeout: f64, @@ -105,6 +121,13 @@ impl Args { "--max-output-tokens and --prompt-bytes must be greater than zero".to_string(), ); } + ScenarioOptions { + model: &self.model, + prompt_bytes: self.prompt_bytes, + max_output_tokens: self.max_output_tokens, + context_window_tokens: self.context_window_tokens, + } + .validate()?; if !self.request_timeout.is_finite() || !self.report_interval.is_finite() || self.request_timeout <= 0.0 @@ -144,8 +167,7 @@ struct RunContext { struct Workload { model: String, - prompts: Vec, - max_output_tokens: u32, + scenarios: Vec, } /// A one-shot stop signal that many tasks can wait on and any task can raise. @@ -194,24 +216,26 @@ async fn worker( context: RunContext, workload: Arc, worker_id: usize, - request_numbers: Arc, ) -> Result<(), String> { + let mut request_number = 0; while !context.stop.is_set() { - let request_number = request_numbers.fetch_add(1, Ordering::Relaxed) as usize; - let endpoint = Endpoint::ALL[request_number % Endpoint::ALL.len()]; - let stream = (request_number / Endpoint::ALL.len()).is_multiple_of(2); - let body = request_body( - endpoint, - &workload.model, - &workload.prompts[request_number % workload.prompts.len()], - workload.max_output_tokens, - stream, - ); + let scenario = &workload.scenarios[request_number % workload.scenarios.len()]; + let stream = (request_number / workload.scenarios.len()).is_multiple_of(2); + let request = scenario.request(request_number / workload.scenarios.len(), stream); + let session_id = format!("{}-worker-{worker_id}", request.session_id); let started = Instant::now(); - let result = send_request(&context.client, &context.base_url, endpoint, &body).await; + let result = send_request( + &context.client, + &context.base_url, + request.endpoint, + &session_id, + request.body, + ) + .await; let latency_ms = started.elapsed().as_secs_f64() * 1000.0; context.stats.lock().record( - endpoint.as_str(), + request.endpoint.as_str(), + scenario.id, latency_ms, result.as_ref().err().map(|error| error.kind.as_str()), ); @@ -222,7 +246,8 @@ async fn worker( .write_error(&json!({ "timestamp_utc": now_utc_string(), "worker": worker_id, - "endpoint": endpoint.as_str(), + "scenario": scenario.id, + "endpoint": request.endpoint.as_str(), "stream": stream, "latency_ms": round3(latency_ms), "error": error.kind, @@ -230,6 +255,7 @@ async fn worker( })) .map_err(|error| error.to_string())?; } + request_number = request_number.wrapping_add(1); } Ok(()) } @@ -306,7 +332,12 @@ fn spawn_signal_listener(stop: Arc) -> tokio::task::JoinHandle<()> { } /// Record every non-secret input with the normalized duration and fixed request variants. -fn write_config(results_dir: &Path, args: &Args, model: &str) -> Result<(), String> { +fn write_config( + results_dir: &Path, + args: &Args, + model: &str, + scenarios: &[Scenario], +) -> Result<(), String> { let config = json!({ "base_url": args.base_url, "model": model, @@ -316,6 +347,8 @@ fn write_config(results_dir: &Path, args: &Args, model: &str) -> Result<(), Stri "streaming": [true, false], "max_output_tokens": args.max_output_tokens, "prompt_bytes": args.prompt_bytes, + "context_window_tokens": args.context_window_tokens, + "scenarios": scenarios.iter().map(|scenario| scenario.id).collect::>(), "request_timeout": args.request_timeout, "report_interval": args.report_interval, "invalid_canary_interval": args.invalid_canary_interval, @@ -333,6 +366,22 @@ fn write_config(results_dir: &Path, args: &Args, model: &str) -> Result<(), Stri pub async fn run(args: Args) -> Result { args.validate()?; + let scenarios = scenarios::select( + ScenarioOptions { + model: &args.model, + prompt_bytes: args.prompt_bytes, + max_output_tokens: args.max_output_tokens, + context_window_tokens: args.context_window_tokens, + }, + args.scenario_set, + &args.scenario, + )?; + if let Some(output_dir) = &args.export_scenarios { + let manifest = scenarios::export(&scenarios, &args.model, output_dir)?; + println!("Exported scenario manifest: {}", manifest.display()); + return Ok(0); + } + let token = match &args.api_key_env { Some(var) => { let token = std::env::var(var).ok().filter(|value| !value.is_empty()); @@ -370,16 +419,16 @@ pub async fn run(args: Args) -> Result { let writer = Arc::new(Mutex::new( ResultsWriter::new(&results_dir).map_err(|error| error.to_string())?, )); - write_config(&results_dir, &args, &args.model)?; + write_config(&results_dir, &args, &args.model, &scenarios)?; println!( - "Soak started: model={} duration={}s concurrency={} endpoints={} results={}", + "Soak started: model={} duration={}s concurrency={} scenarios={} results={}", args.model, args.duration, args.concurrency, - Endpoint::ALL + scenarios .iter() - .map(|endpoint| endpoint.as_str()) + .map(|scenario| scenario.id) .collect::>() .join(","), results_dir.display(), @@ -389,7 +438,6 @@ pub async fn run(args: Args) -> Result { let stats = Arc::new(Mutex::new(RunStats::new(2026))); let stop = Arc::new(Stop::new()); let workers_done = Arc::new(Stop::new()); - let request_numbers = Arc::new(AtomicU64::new(0)); let context = RunContext { client, base_url, @@ -413,19 +461,13 @@ pub async fn run(args: Args) -> Result { let workload = Arc::new(Workload { model: args.model.clone(), - prompts: build_prompt_pool(args.prompt_bytes), - max_output_tokens: args.max_output_tokens, + scenarios, }); let mut worker_handles = Vec::new(); for worker_id in 0..args.concurrency { let handle = tokio::spawn(guard_stop( stop.clone(), - worker( - context.clone(), - workload.clone(), - worker_id, - request_numbers.clone(), - ), + worker(context.clone(), workload.clone(), worker_id), )); worker_handles.push((format!("worker-{worker_id}"), handle)); } diff --git a/crates/switchyard-soak/src/scenarios/classifier_mix.rs b/crates/switchyard-soak/src/scenarios/classifier_mix.rs new file mode 100644 index 000000000..893366fdf --- /dev/null +++ b/crates/switchyard-soak/src/scenarios/classifier_mix.rs @@ -0,0 +1,42 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use super::{ + ErrorExpectation, Scenario, ScenarioGroup, ScenarioOptions, Session, chat_payload, user, +}; + +pub fn build(options: ScenarioOptions<'_>) -> Scenario { + let sessions = (0..20) + .map(|index| { + let hard = (8..10).contains(&index) || index >= 15; + let marker = if hard { + "classifier_hard" + } else { + "classifier_easy" + }; + Session { + session_id: format!("classifier-mix-{index}"), + payloads: vec![chat_payload( + options.model, + vec![user(format!( + "[scenario:{marker}] Request {index}: {}", + if hard { + "reason about a distributed failure with incomplete evidence" + } else { + "return the sum of two and two" + } + ))], + options.max_output_tokens, + )], + } + }) + .collect(); + Scenario::chat( + "classifier-mix", + ScenarioGroup::Agentic, + "Two deterministic easy/hard mixes: 80/20 followed by 50/50.", + "Selection shares match the input mix and classifier overhead is visible.", + ErrorExpectation::SUCCESS, + sessions, + ) +} diff --git a/crates/switchyard-soak/src/scenarios/client_cancellation.rs b/crates/switchyard-soak/src/scenarios/client_cancellation.rs new file mode 100644 index 000000000..66ccf29e3 --- /dev/null +++ b/crates/switchyard-soak/src/scenarios/client_cancellation.rs @@ -0,0 +1,26 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use super::{ + ErrorExpectation, Scenario, ScenarioGroup, ScenarioOptions, Session, chat_payload, user, +}; + +pub fn build(options: ScenarioOptions<'_>) -> Scenario { + Scenario::chat( + "client-cancellation", + ScenarioGroup::Resilience, + "Slow streaming requests whose client timeout cancels work in flight.", + "Cancellation releases connections and does not destabilize later traffic.", + ErrorExpectation::ALL, + vec![Session { + session_id: "client-cancellation".to_string(), + payloads: vec![chat_payload( + options.model, + vec![user( + "[scenario:client_cancellation] Delay this response until the client leaves.", + )], + options.max_output_tokens, + )], + }], + ) +} diff --git a/crates/switchyard-soak/src/scenarios/context_overflow.rs b/crates/switchyard-soak/src/scenarios/context_overflow.rs new file mode 100644 index 000000000..3e57e2007 --- /dev/null +++ b/crates/switchyard-soak/src/scenarios/context_overflow.rs @@ -0,0 +1,26 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use super::{ + ErrorExpectation, Scenario, ScenarioGroup, ScenarioOptions, Session, chat_payload, token_text, + user, +}; + +pub fn build(options: ScenarioOptions<'_>) -> Scenario { + let tokens = (options.context_window_tokens * 9 / 10).max(1); + Scenario::chat( + "context-overflow", + ScenarioGroup::Resilience, + "A near-window request that one configured target rejects as too long.", + "A multi-target route retries an eligible target and returns a valid response.", + ErrorExpectation::SUCCESS, + vec![Session { + session_id: "context-overflow".to_string(), + payloads: vec![chat_payload( + options.model, + vec![user(token_text(tokens, "context_overflow"))], + options.max_output_tokens, + )], + }], + ) +} diff --git a/crates/switchyard-soak/src/scenarios/decode_heavy.rs b/crates/switchyard-soak/src/scenarios/decode_heavy.rs new file mode 100644 index 000000000..b19c4d68a --- /dev/null +++ b/crates/switchyard-soak/src/scenarios/decode_heavy.rs @@ -0,0 +1,30 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use super::{ + ErrorExpectation, Scenario, ScenarioGroup, ScenarioOptions, Session, chat_payload, user, +}; + +pub fn build(options: ScenarioOptions<'_>) -> Scenario { + let sessions = [512, 1024] + .into_iter() + .map(|tokens| Session { + session_id: format!("decode-heavy-{tokens}"), + payloads: vec![chat_payload( + options.model, + vec![user(format!( + "[scenario:decode_heavy] Write {tokens} numbered one-word items." + ))], + tokens, + )], + }) + .collect(); + Scenario::chat( + "decode-heavy", + ScenarioGroup::Core, + "Short inputs with 512-token and 1,024-token output limits.", + "Output token throughput stays stable while long streams remain valid.", + ErrorExpectation::SUCCESS, + sessions, + ) +} diff --git a/crates/switchyard-soak/src/scenarios/failure_pressure.rs b/crates/switchyard-soak/src/scenarios/failure_pressure.rs new file mode 100644 index 000000000..a1606b065 --- /dev/null +++ b/crates/switchyard-soak/src/scenarios/failure_pressure.rs @@ -0,0 +1,36 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use super::{ + ErrorExpectation, Scenario, ScenarioGroup, ScenarioOptions, Session, chat_payload, user, +}; + +pub fn build(options: ScenarioOptions<'_>) -> Scenario { + let cases = [ + ("truncated-stream", "truncated_stream"), + ("upstream-429", "upstream_429"), + ("upstream-500", "upstream_500"), + ("classifier-invalid", "classifier_invalid"), + ]; + let sessions = cases + .into_iter() + .map(|(id, marker)| Session { + session_id: id.to_string(), + payloads: vec![chat_payload( + options.model, + vec![user(format!( + "[scenario:{marker}] Exercise the configured recovery path." + ))], + options.max_output_tokens, + )], + }) + .collect(); + Scenario::chat( + "failure-pressure", + ScenarioGroup::Resilience, + "Bounded 429, 500, malformed-classifier, and truncated-stream injections.", + "Retries recover transient failures; terminal failures remain explicit and bounded.", + ErrorExpectation::MIXED, + sessions, + ) +} diff --git a/crates/switchyard-soak/src/scenarios/growing_conversation.rs b/crates/switchyard-soak/src/scenarios/growing_conversation.rs new file mode 100644 index 000000000..17da931aa --- /dev/null +++ b/crates/switchyard-soak/src/scenarios/growing_conversation.rs @@ -0,0 +1,34 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use super::{ + ErrorExpectation, Scenario, ScenarioGroup, ScenarioOptions, Session, assistant, chat_payload, + user, +}; + +pub fn build(options: ScenarioOptions<'_>) -> Scenario { + let mut messages = Vec::new(); + let mut payloads = Vec::new(); + for turn in 0..8 { + messages.push(user(format!( + "[scenario:growing_conversation] Turn {turn}: remember the number {turn}." + ))); + payloads.push(chat_payload( + options.model, + messages.clone(), + options.max_output_tokens.min(64), + )); + messages.push(assistant(format!("I recorded {turn}."))); + } + Scenario::chat( + "growing-conversation", + ScenarioGroup::Agentic, + "An eight-turn session whose complete message history grows each turn.", + "Session affinity holds and latency tracks the growing history.", + ErrorExpectation::SUCCESS, + vec![Session { + session_id: "growing-conversation".to_string(), + payloads, + }], + ) +} diff --git a/crates/switchyard-soak/src/scenarios/large_tool_catalog.rs b/crates/switchyard-soak/src/scenarios/large_tool_catalog.rs new file mode 100644 index 000000000..a95363f18 --- /dev/null +++ b/crates/switchyard-soak/src/scenarios/large_tool_catalog.rs @@ -0,0 +1,61 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use serde_json::{Value, json}; + +use super::{ + ErrorExpectation, Scenario, ScenarioGroup, ScenarioOptions, Session, chat_payload, user, +}; + +fn tools(count: usize) -> Vec { + (0..count) + .map(|index| { + json!({ + "type": "function", + "function": { + "name": format!("lookup_record_{index}"), + "description": format!("Look up record {index} in the test catalog."), + "parameters": { + "type": "object", + "properties": { + "query": {"type": "string"}, + "limit": {"type": "integer", "minimum": 1, "maximum": 10} + }, + "required": ["query"], + "additionalProperties": false + } + } + }) + }) + .collect() +} + +pub fn build(options: ScenarioOptions<'_>) -> Scenario { + let sessions = [16, 64] + .into_iter() + .map(|count| { + let mut payload = chat_payload( + options.model, + vec![user(format!( + "[scenario:large_tool_catalog] Use lookup_record_{} for query green.", + count - 1 + ))], + options.max_output_tokens.min(128), + ); + payload["tools"] = Value::Array(tools(count)); + payload["tool_choice"] = json!("auto"); + Session { + session_id: format!("large-tool-catalog-{count}"), + payloads: vec![payload], + } + }) + .collect(); + Scenario::chat( + "large-tool-catalog", + ScenarioGroup::Agentic, + "Requests with 16-tool and 64-tool JSON-schema catalogs.", + "Routing overhead remains bounded while large tool schemas are forwarded intact.", + ErrorExpectation::SUCCESS, + sessions, + ) +} diff --git a/crates/switchyard-soak/src/scenarios/long_context.rs b/crates/switchyard-soak/src/scenarios/long_context.rs new file mode 100644 index 000000000..6958e847f --- /dev/null +++ b/crates/switchyard-soak/src/scenarios/long_context.rs @@ -0,0 +1,36 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use super::{ + ErrorExpectation, Scenario, ScenarioGroup, ScenarioOptions, Session, chat_payload, token_text, + user, +}; + +pub fn build(options: ScenarioOptions<'_>) -> Scenario { + let mut lengths = vec![8_192, 32_000, options.context_window_tokens * 9 / 10]; + lengths.retain(|length| *length < options.context_window_tokens); + lengths.sort_unstable(); + lengths.dedup(); + let sessions = lengths + .into_iter() + .map(|tokens| Session { + session_id: format!("long-context-{tokens}"), + payloads: vec![chat_payload( + options.model, + vec![user(format!( + "{}\nSummarize the final marker in one sentence.", + token_text(tokens, "long_context") + ))], + options.max_output_tokens.min(64), + )], + }) + .collect(); + Scenario::chat( + "long-context", + ScenarioGroup::Core, + "Requests at 8K, 32K, and near the configured context window.", + "TTFT rises with input length without routing errors or memory growth.", + ErrorExpectation::SUCCESS, + sessions, + ) +} diff --git a/crates/switchyard-soak/src/scenarios/mixed_traffic.rs b/crates/switchyard-soak/src/scenarios/mixed_traffic.rs new file mode 100644 index 000000000..ae1a444a4 --- /dev/null +++ b/crates/switchyard-soak/src/scenarios/mixed_traffic.rs @@ -0,0 +1,41 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use super::{ + ErrorExpectation, Scenario, ScenarioGroup, ScenarioOptions, Session, chat_payload, token_text, + user, +}; + +pub fn build(options: ScenarioOptions<'_>) -> Scenario { + let lengths = [16, 24, 32, 48, 64, 96, 128, 1_024, 2_048, 8_192]; + let sessions = lengths + .into_iter() + .enumerate() + .map(|(index, tokens)| Session { + session_id: format!("mixed-{index}"), + payloads: vec![chat_payload( + options.model, + vec![user(format!( + "{}\nReply with the request number {index}.", + token_text( + tokens.min(options.context_window_tokens / 2), + "mixed_traffic" + ) + ))], + if index == 9 { + 256 + } else { + options.max_output_tokens + }, + )], + }) + .collect(); + Scenario::chat( + "mixed-traffic", + ScenarioGroup::Core, + "A 70/20/10 mix of short, medium, and long requests.", + "Tail latency stays bounded when unlike request sizes share one route.", + ErrorExpectation::SUCCESS, + sessions, + ) +} diff --git a/crates/switchyard-soak/src/scenarios/mod.rs b/crates/switchyard-soak/src/scenarios/mod.rs new file mode 100644 index 000000000..b7bf9a2cf --- /dev/null +++ b/crates/switchyard-soak/src/scenarios/mod.rs @@ -0,0 +1,528 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Reproducible request scenarios shared by the soak command and AIPerf runner. + +mod classifier_mix; +mod client_cancellation; +mod context_overflow; +mod decode_heavy; +mod failure_pressure; +mod growing_conversation; +mod large_tool_catalog; +mod long_context; +mod mixed_traffic; +mod prefix_reuse; +mod short_interactive; +mod stage_transitions; +mod tool_call_burst; + +use std::collections::HashSet; +use std::fs; +use std::path::{Path, PathBuf}; + +use clap::ValueEnum; +use serde::Serialize; +use serde_json::{Value, json}; + +use crate::client::Endpoint; + +/// Largest accepted context-window input for generated scenario data. +const MAX_CONTEXT_WINDOW_TOKENS: usize = 1_000_000; +const MIN_CONTEXT_WINDOW_TOKENS: usize = 16_384; +const MAX_PROMPT_BYTES: usize = 1_000_000; + +/// Scenario family used by CLI selection and report grouping. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ScenarioGroup { + Core, + Agentic, + Resilience, +} + +/// Accepted client-visible error-rate range for one scenario. +#[derive(Clone, Copy, Serialize)] +pub struct ErrorExpectation { + pub min_rate: f64, + pub max_rate: f64, +} + +impl ErrorExpectation { + pub const SUCCESS: Self = Self { + min_rate: 0.0, + max_rate: 0.0, + }; + pub const MIXED: Self = Self { + min_rate: 0.01, + max_rate: 0.75, + }; + pub const ALL: Self = Self { + min_rate: 0.8, + max_rate: 1.0, + }; +} + +/// Named scenario collection selected by the soak and benchmark commands. +#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] +#[value(rename_all = "snake_case")] +pub enum ScenarioSet { + Core, + Agentic, + Resilience, + Standard, + All, +} + +impl ScenarioSet { + fn contains(self, group: ScenarioGroup) -> bool { + match self { + Self::Core => group == ScenarioGroup::Core, + Self::Agentic => group == ScenarioGroup::Agentic, + Self::Resilience => group == ScenarioGroup::Resilience, + Self::Standard => group != ScenarioGroup::Resilience, + Self::All => true, + } + } +} + +/// Request-generation limits shared by every scenario builder. +#[derive(Clone, Copy)] +pub struct ScenarioOptions<'a> { + pub model: &'a str, + pub prompt_bytes: usize, + pub max_output_tokens: u32, + pub context_window_tokens: usize, +} + +impl ScenarioOptions<'_> { + pub fn validate(&self) -> Result<(), String> { + if self.model.is_empty() { + return Err("scenario model must not be empty".to_string()); + } + if self.prompt_bytes == 0 || self.max_output_tokens == 0 { + return Err("scenario prompt and output limits must be greater than zero".to_string()); + } + if !(MIN_CONTEXT_WINDOW_TOKENS..=MAX_CONTEXT_WINDOW_TOKENS) + .contains(&self.context_window_tokens) + { + return Err(format!( + "--context-window-tokens must be between {MIN_CONTEXT_WINDOW_TOKENS} and {MAX_CONTEXT_WINDOW_TOKENS}" + )); + } + let prompt_limit = MAX_PROMPT_BYTES.min(self.context_window_tokens.saturating_mul(2)); + if self.prompt_bytes > prompt_limit { + return Err(format!( + "--prompt-bytes must not exceed {prompt_limit} for this context window" + )); + } + Ok(()) + } +} + +/// One AIPerf conversation; payloads remain ordered within the session. +#[derive(Clone, Serialize)] +pub struct Session { + pub session_id: String, + pub payloads: Vec, +} + +/// One request the Rust soak command can send. +pub struct RequestCase { + pub endpoint: Endpoint, + pub session_id: String, + streaming_body: Value, + nonstreaming_body: Value, +} + +impl RequestCase { + pub fn new(endpoint: Endpoint, session_id: String, streaming_body: Value) -> Self { + let mut nonstreaming_body = streaming_body.clone(); + nonstreaming_body["stream"] = Value::Bool(false); + Self { + endpoint, + session_id, + streaming_body, + nonstreaming_body, + } + } +} + +pub struct PreparedRequest<'a> { + pub endpoint: Endpoint, + pub session_id: &'a str, + pub body: &'a Value, +} + +/// QPS point expressed relative to the benchmark command's base request rate. +#[derive(Clone, Serialize)] +pub struct RatePoint { + pub time_s: u32, + pub rate_multiplier: f64, +} + +/// AIPerf schedule applied to a request scenario. +#[derive(Clone, Serialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum LoadProfile { + Fixed { + id: &'static str, + }, + ConcurrencyKnee { + id: &'static str, + concurrency_steps: Vec, + }, + TrafficBurst { + id: &'static str, + duration_seconds: u32, + points: Vec, + }, +} + +/// Immutable scenario data built once before load workers start. +pub struct Scenario { + pub id: &'static str, + pub group: ScenarioGroup, + pub description: &'static str, + pub expected: &'static str, + pub expected_error_rate: ErrorExpectation, + pub sessions: Vec, + pub soak_requests: Vec, + pub load_profiles: Vec, +} + +impl Scenario { + /// Build a Chat Completions scenario whose AIPerf payloads also drive soak traffic. + pub fn chat( + id: &'static str, + group: ScenarioGroup, + description: &'static str, + expected: &'static str, + expected_error_rate: ErrorExpectation, + sessions: Vec, + ) -> Self { + let soak_requests = + sessions + .iter() + .flat_map(|session| { + session.payloads.iter().cloned().map(|body| { + RequestCase::new(Endpoint::Chat, session.session_id.clone(), body) + }) + }) + .collect(); + Self { + id, + group, + description, + expected, + expected_error_rate, + sessions, + soak_requests, + load_profiles: fixed_load(), + } + } + + /// Return one prebuilt request with only its streaming flag changed. + pub fn request(&self, index: usize, stream: bool) -> PreparedRequest<'_> { + let selected = &self.soak_requests[index % self.soak_requests.len()]; + PreparedRequest { + endpoint: selected.endpoint, + session_id: &selected.session_id, + body: if stream { + &selected.streaming_body + } else { + &selected.nonstreaming_body + }, + } + } +} + +/// Return the one fixed schedule used by ordinary request scenarios. +pub fn fixed_load() -> Vec { + vec![LoadProfile::Fixed { id: "fixed" }] +} + +/// Build the three bounded load schedules used for the short baseline. +pub fn baseline_load() -> Vec { + vec![ + LoadProfile::Fixed { id: "fixed" }, + LoadProfile::ConcurrencyKnee { + id: "concurrency-knee", + concurrency_steps: vec![1, 4, 16, 64, 128], + }, + LoadProfile::TrafficBurst { + id: "traffic-burst", + duration_seconds: 30, + points: vec![ + RatePoint { + time_s: 0, + rate_multiplier: 1.0, + }, + RatePoint { + time_s: 10, + rate_multiplier: 1.0, + }, + RatePoint { + time_s: 11, + rate_multiplier: 10.0, + }, + RatePoint { + time_s: 16, + rate_multiplier: 10.0, + }, + RatePoint { + time_s: 17, + rate_multiplier: 1.0, + }, + RatePoint { + time_s: 30, + rate_multiplier: 1.0, + }, + ], + }, + ] +} + +/// Build one raw Chat Completions payload. +pub fn chat_payload(model: &str, messages: Vec, max_tokens: u32) -> Value { + json!({ + "model": model, + "messages": messages, + "max_tokens": max_tokens, + "temperature": 0, + "stream": true, + }) +} + +/// Build one user message. +pub fn user(content: impl Into) -> Value { + json!({"role": "user", "content": content.into()}) +} + +/// Build one assistant message. +pub fn assistant(content: impl Into) -> Value { + json!({"role": "assistant", "content": content.into()}) +} + +/// Build repeated context with a stable approximate token count. +pub fn token_text(tokens: usize, label: &str) -> String { + format!("[scenario:{label}] {}", "x ".repeat(tokens)) +} + +/// Build the complete catalog in stable report order. +pub fn catalog(options: ScenarioOptions<'_>) -> Result, String> { + options.validate()?; + Ok(vec![ + short_interactive::build(options), + long_context::build(options), + decode_heavy::build(options), + prefix_reuse::build(options), + mixed_traffic::build(options), + growing_conversation::build(options), + large_tool_catalog::build(options), + tool_call_burst::build(options), + stage_transitions::build(options), + classifier_mix::build(options), + context_overflow::build(options), + failure_pressure::build(options), + client_cancellation::build(options), + ]) +} + +/// Select explicit scenario IDs or every scenario in a named set. +pub fn select( + options: ScenarioOptions<'_>, + set: ScenarioSet, + requested: &[String], +) -> Result, String> { + let mut catalog = catalog(options)?; + if requested.is_empty() { + return Ok(catalog + .into_iter() + .filter(|scenario| set.contains(scenario.group)) + .collect()); + } + + let mut seen = HashSet::new(); + let mut selected = Vec::new(); + for id in requested { + if !seen.insert(id) { + return Err(format!("scenario {id:?} was requested more than once")); + } + let index = catalog + .iter() + .position(|scenario| scenario.id == id) + .ok_or_else(|| format!("unknown scenario {id:?}"))?; + selected.push(catalog.remove(index)); + } + Ok(selected) +} + +#[derive(Serialize)] +struct InputsFile<'a> { + data: &'a [Session], +} + +#[derive(Serialize)] +struct Manifest<'a> { + schema_version: u32, + model: &'a str, + scenarios: Vec>, +} + +#[derive(Serialize)] +struct ManifestScenario<'a> { + id: &'a str, + group: ScenarioGroup, + description: &'a str, + expected: &'a str, + expected_error_rate: ErrorExpectation, + input_file: String, + load_profiles: &'a [LoadProfile], +} + +/// Export the selected scenario catalog in AIPerf's verbatim inputs JSON format. +pub fn export(scenarios: &[Scenario], model: &str, output_dir: &Path) -> Result { + if output_dir.exists() { + return Err(format!( + "scenario output directory already exists: {}", + output_dir.display() + )); + } + let inputs_dir = output_dir.join("inputs"); + fs::create_dir_all(&inputs_dir).map_err(|error| error.to_string())?; + + let mut entries = Vec::new(); + for scenario in scenarios { + let relative = format!("inputs/{}.json", scenario.id); + let path = output_dir.join(&relative); + let body = serde_json::to_string_pretty(&InputsFile { + data: &scenario.sessions, + }) + .map_err(|error| error.to_string())?; + fs::write(path, format!("{body}\n")).map_err(|error| error.to_string())?; + entries.push(ManifestScenario { + id: scenario.id, + group: scenario.group, + description: scenario.description, + expected: scenario.expected, + expected_error_rate: scenario.expected_error_rate, + input_file: relative, + load_profiles: &scenario.load_profiles, + }); + } + + let manifest_path = output_dir.join("manifest.json"); + let body = serde_json::to_string_pretty(&Manifest { + schema_version: 1, + model, + scenarios: entries, + }) + .map_err(|error| error.to_string())?; + fs::write(&manifest_path, format!("{body}\n")).map_err(|error| error.to_string())?; + Ok(manifest_path) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn options() -> ScenarioOptions<'static> { + ScenarioOptions { + model: "switchyard/test", + prompt_bytes: 128, + max_output_tokens: 32, + context_window_tokens: 16_384, + } + } + + #[test] + fn catalog_is_unique_bounded_and_serializable() -> Result<(), String> { + let scenarios = catalog(options())?; + let names = scenarios + .iter() + .map(|scenario| scenario.id) + .collect::>(); + assert_eq!(names.len(), scenarios.len()); + assert!(scenarios.iter().all(|scenario| { + !scenario.sessions.is_empty() + && !scenario.soak_requests.is_empty() + && scenario.sessions.len() <= 32 + && scenario + .sessions + .iter() + .all(|session| !session.payloads.is_empty() && session.payloads.len() <= 32) + })); + + let parent = tempfile::tempdir().map_err(|error| error.to_string())?; + let output = parent.path().join("export"); + let manifest = export(&scenarios, options().model, &output)?; + let payload: Value = + serde_json::from_str(&fs::read_to_string(manifest).map_err(|error| error.to_string())?) + .map_err(|error| error.to_string())?; + assert_eq!(payload["scenarios"].as_array().map(Vec::len), Some(13)); + Ok(()) + } + + #[test] + fn explicit_selection_keeps_order_and_rejects_bad_names() -> Result<(), String> { + let requested = vec!["tool-call-burst".to_string(), "long-context".to_string()]; + let selected = select(options(), ScenarioSet::Core, &requested)?; + assert_eq!( + selected + .iter() + .map(|scenario| scenario.id) + .collect::>(), + ["tool-call-burst", "long-context"] + ); + assert!(select(options(), ScenarioSet::Core, &["missing".to_string()]).is_err()); + Ok(()) + } + + #[test] + fn agentic_payloads_keep_their_bounded_contracts() -> Result<(), String> { + let scenarios = catalog(options())?; + let find = |id| { + scenarios + .iter() + .find(|scenario| scenario.id == id) + .ok_or_else(|| format!("missing scenario {id}")) + }; + + let long = find("long-context")?; + assert!(long.sessions.iter().any(|session| { + session.payloads[0]["messages"][0]["content"] + .as_str() + .is_some_and(|content| content.len() > 8_000) + })); + + let tools = find("large-tool-catalog")?; + assert_eq!( + tools.sessions[1].payloads[0]["tools"] + .as_array() + .map(Vec::len), + Some(64) + ); + + let burst = find("tool-call-burst")?; + assert_eq!(burst.sessions[0].payloads.len(), 9); + let final_messages = burst.sessions[0].payloads[8]["messages"] + .as_array() + .ok_or_else(|| "tool-call burst messages were not an array".to_string())?; + assert!( + final_messages.iter().any(|message| { + message["role"] == "tool" && message["tool_call_id"] == "call_7" + }) + ); + + let growing = find("growing-conversation")?; + let message_counts = growing.sessions[0] + .payloads + .iter() + .map(|payload| payload["messages"].as_array().map(Vec::len).unwrap_or(0)) + .collect::>(); + assert!(message_counts.windows(2).all(|pair| pair[0] < pair[1])); + assert!(find("failure-pressure")?.expected_error_rate.min_rate > 0.0); + Ok(()) + } +} diff --git a/crates/switchyard-soak/src/scenarios/prefix_reuse.rs b/crates/switchyard-soak/src/scenarios/prefix_reuse.rs new file mode 100644 index 000000000..7272ec501 --- /dev/null +++ b/crates/switchyard-soak/src/scenarios/prefix_reuse.rs @@ -0,0 +1,37 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use super::{ + ErrorExpectation, Scenario, ScenarioGroup, ScenarioOptions, Session, chat_payload, token_text, + user, +}; + +pub fn build(options: ScenarioOptions<'_>) -> Scenario { + let prefix_tokens = (options.prompt_bytes / 2).clamp(512, 8_192); + let shared = token_text(prefix_tokens, "prefix_reuse_shared"); + let sessions = (0..8) + .map(|index| { + let prefix = if index < 4 { + shared.clone() + } else { + token_text(prefix_tokens, &format!("prefix_reuse_unique_{index}")) + }; + Session { + session_id: format!("prefix-reuse-{index}"), + payloads: vec![chat_payload( + options.model, + vec![user(format!("{prefix}\nReturn item {index}."))], + options.max_output_tokens.min(32), + )], + } + }) + .collect(); + Scenario::chat( + "prefix-reuse", + ScenarioGroup::Core, + "Matched requests with shared and unique long prefixes.", + "The report exposes cache-sensitive TTFT without changing route behavior.", + ErrorExpectation::SUCCESS, + sessions, + ) +} diff --git a/crates/switchyard-soak/src/scenarios/short_interactive.rs b/crates/switchyard-soak/src/scenarios/short_interactive.rs new file mode 100644 index 000000000..023f04161 --- /dev/null +++ b/crates/switchyard-soak/src/scenarios/short_interactive.rs @@ -0,0 +1,61 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use crate::client::{Endpoint, request_body}; + +use super::{ + ErrorExpectation, RequestCase, Scenario, ScenarioGroup, ScenarioOptions, Session, + baseline_load, chat_payload, user, +}; + +pub fn build(options: ScenarioOptions<'_>) -> Scenario { + let prompts = [ + "Reply with exactly OK.".to_string(), + "Name one primary color.".to_string(), + "Return the number after 41.".to_string(), + format!( + "[scenario:short_interactive] {} Reply briefly.", + "stable prefix ".repeat(options.prompt_bytes.div_ceil(14)) + ), + ]; + let sessions = prompts + .iter() + .enumerate() + .map(|(index, prompt)| Session { + session_id: format!("short-{index}"), + payloads: vec![chat_payload( + options.model, + vec![user(prompt)], + options.max_output_tokens, + )], + }) + .collect(); + let soak_requests = Endpoint::ALL + .iter() + .flat_map(|endpoint| { + prompts.iter().enumerate().map(move |(index, prompt)| { + RequestCase::new( + *endpoint, + format!("short-{index}-{}", endpoint.as_str()), + request_body( + *endpoint, + options.model, + prompt, + options.max_output_tokens, + true, + ), + ) + }) + }) + .collect(); + Scenario { + id: "short-interactive", + group: ScenarioGroup::Core, + description: "Short prompts that establish HTTP, routing, TTFT, and latency overhead.", + expected: "All public endpoints succeed; oha and AIPerf establish the baseline.", + expected_error_rate: ErrorExpectation::SUCCESS, + sessions, + soak_requests, + load_profiles: baseline_load(), + } +} diff --git a/crates/switchyard-soak/src/scenarios/stage_transitions.rs b/crates/switchyard-soak/src/scenarios/stage_transitions.rs new file mode 100644 index 000000000..63916e37e --- /dev/null +++ b/crates/switchyard-soak/src/scenarios/stage_transitions.rs @@ -0,0 +1,53 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use serde_json::json; + +use super::{ + ErrorExpectation, Scenario, ScenarioGroup, ScenarioOptions, Session, chat_payload, user, +}; + +pub fn build(options: ScenarioOptions<'_>) -> Scenario { + let mut messages = vec![user( + "[scenario:stage_exploration] Find the failing module, then fix it.", + )]; + let mut payloads = vec![chat_payload( + options.model, + messages.clone(), + options.max_output_tokens.min(128), + )]; + + messages.extend([ + json!({"role":"assistant","tool_calls":[{"id":"stage_1","type":"function","function":{"name":"run_tests","arguments":"{}"}}]}), + json!({"role":"tool","tool_call_id":"stage_1","content":"fatal: critical tool failure: repository index is unavailable"}), + ]); + payloads.push(chat_payload( + options.model, + messages.clone(), + options.max_output_tokens.min(128), + )); + + messages.extend([ + json!({"role":"assistant","tool_calls":[{"id":"stage_2","type":"function","function":{"name":"apply_patch","arguments":"{}"}}]}), + json!({"role":"tool","tool_call_id":"stage_2","content":"patch applied"}), + json!({"role":"assistant","tool_calls":[{"id":"stage_3","type":"function","function":{"name":"run_tests","arguments":"{}"}}]}), + json!({"role":"tool","tool_call_id":"stage_3","content":"all tests passed"}), + ]); + payloads.push(chat_payload( + options.model, + messages, + options.max_output_tokens.min(128), + )); + + Scenario::chat( + "stage-transitions", + ScenarioGroup::Agentic, + "One growing session that moves through exploration, critical failure, and productive work.", + "The stage router changes tiers at the configured evidence boundaries.", + ErrorExpectation::SUCCESS, + vec![Session { + session_id: "stage-transitions".to_string(), + payloads, + }], + ) +} diff --git a/crates/switchyard-soak/src/scenarios/tool_call_burst.rs b/crates/switchyard-soak/src/scenarios/tool_call_burst.rs new file mode 100644 index 000000000..1c6b35d69 --- /dev/null +++ b/crates/switchyard-soak/src/scenarios/tool_call_burst.rs @@ -0,0 +1,65 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use serde_json::{Value, json}; + +use super::{ + ErrorExpectation, Scenario, ScenarioGroup, ScenarioOptions, Session, chat_payload, user, +}; + +pub fn build(options: ScenarioOptions<'_>) -> Scenario { + let mut messages = vec![user( + "[scenario:tool_call_burst] Inspect eight shards, one call at a time.", + )]; + let mut payloads = vec![chat_payload( + options.model, + messages.clone(), + options.max_output_tokens.min(128), + )]; + for index in 0..8 { + let call_id = format!("call_{index}"); + messages.push(json!({ + "role": "assistant", + "content": null, + "tool_calls": [{ + "id": call_id, + "type": "function", + "function": {"name": "inspect_shard", "arguments": format!("{{\"shard\":{index}}}")} + }] + })); + messages.push(json!({ + "role": "tool", + "tool_call_id": format!("call_{index}"), + "content": format!("shard {index}: healthy") + })); + payloads.push(chat_payload( + options.model, + messages.clone(), + options.max_output_tokens.min(128), + )); + } + for payload in &mut payloads { + payload["tools"] = Value::Array(vec![json!({ + "type": "function", + "function": { + "name": "inspect_shard", + "parameters": { + "type": "object", + "properties": {"shard": {"type": "integer"}}, + "required": ["shard"] + } + } + })]); + } + Scenario::chat( + "tool-call-burst", + ScenarioGroup::Agentic, + "An eight-turn burst of linked assistant tool calls and tool results.", + "The route keeps session state and forwards every call/result pair.", + ErrorExpectation::SUCCESS, + vec![Session { + session_id: "tool-call-burst".to_string(), + payloads, + }], + ) +} diff --git a/crates/switchyard-soak/src/stats.rs b/crates/switchyard-soak/src/stats.rs index e5b38412d..dcb8f1f20 100644 --- a/crates/switchyard-soak/src/stats.rs +++ b/crates/switchyard-soak/src/stats.rs @@ -60,20 +60,6 @@ pub fn latency_stats(values: &mut [f64]) -> LatencyStats { } } -/// Return four stable prompts with reusable prefixes of *prompt_bytes* filler each. -pub fn build_prompt_pool(prompt_bytes: usize) -> Vec { - (0..4) - .map(|index| { - let prefix = format!("Switchyard soak prefix {index}. "); - let instruction = "Reply with exactly OK. "; - let unit = "load test context "; - let mut filler = unit.repeat(prompt_bytes / unit.len() + 1); - filler.truncate(prompt_bytes); - format!("{prefix}{filler}{instruction}") - }) - .collect() -} - /// ISO-8601 UTC timestamp at second precision, e.g. `2026-07-30T18:04:00Z`. pub fn now_utc_string() -> String { humantime::format_rfc3339_seconds(SystemTime::now()).to_string() @@ -103,6 +89,8 @@ pub struct RunStats { pub total_failures: u64, pub endpoint_successes: BTreeMap, pub endpoint_failures: BTreeMap, + pub scenario_successes: BTreeMap, + pub scenario_failures: BTreeMap, pub error_kinds: BTreeMap, latency_reservoir: Vec, latency_count: u64, @@ -129,6 +117,8 @@ impl RunStats { total_failures: 0, endpoint_successes: BTreeMap::new(), endpoint_failures: BTreeMap::new(), + scenario_successes: BTreeMap::new(), + scenario_failures: BTreeMap::new(), error_kinds: BTreeMap::new(), latency_reservoir: Vec::new(), latency_count: 0, @@ -148,7 +138,13 @@ impl RunStats { } /// Record one completed inference request; `error_kind` is `None` on success. - pub fn record(&mut self, endpoint: &str, latency_ms: f64, error_kind: Option<&str>) { + pub fn record( + &mut self, + endpoint: &str, + scenario: &str, + latency_ms: f64, + error_kind: Option<&str>, + ) { match error_kind { None => { self.interval.successes += 1; @@ -157,6 +153,10 @@ impl RunStats { .endpoint_successes .entry(endpoint.to_string()) .or_default() += 1; + *self + .scenario_successes + .entry(scenario.to_string()) + .or_default() += 1; } Some(kind) => { self.interval.failures += 1; @@ -165,6 +165,10 @@ impl RunStats { .endpoint_failures .entry(endpoint.to_string()) .or_default() += 1; + *self + .scenario_failures + .entry(scenario.to_string()) + .or_default() += 1; *self.error_kinds.entry(kind.to_string()).or_default() += 1; } } @@ -200,6 +204,8 @@ pub struct Summary { pub requests_per_second: f64, pub endpoint_successes: BTreeMap, pub endpoint_failures: BTreeMap, + pub scenario_successes: BTreeMap, + pub scenario_failures: BTreeMap, pub error_kinds: BTreeMap, pub latency_p50_ms: Option, pub latency_p95_ms: Option, @@ -321,6 +327,8 @@ pub fn build_summary( requests_per_second, endpoint_successes: stats.endpoint_successes.clone(), endpoint_failures: stats.endpoint_failures.clone(), + scenario_successes: stats.scenario_successes.clone(), + scenario_failures: stats.scenario_failures.clone(), error_kinds: stats.error_kinds.clone(), latency_p50_ms: latency.p50_ms, latency_p95_ms: latency.p95_ms, @@ -375,8 +383,8 @@ mod tests { #[test] fn record_tracks_interval_and_cumulative_results() { let mut stats = RunStats::new(1); - stats.record("chat", 10.0, None); - stats.record("messages", 20.0, Some("timeout")); + stats.record("chat", "short-interactive", 10.0, None); + stats.record("messages", "long-context", 20.0, Some("timeout")); let interval = stats.take_interval(); assert_eq!(interval.successes, 1); @@ -390,7 +398,7 @@ mod tests { fn latency_samples_stay_bounded_after_the_reservoir_fills() { let mut stats = RunStats::new(1); for sample in 0..=RESERVOIR_SIZE { - stats.record("chat", sample as f64, None); + stats.record("chat", "short-interactive", sample as f64, None); } assert_eq!(stats.latency_reservoir.len(), RESERVOIR_SIZE); diff --git a/crates/switchyard-soak/tests/soak.rs b/crates/switchyard-soak/tests/soak.rs index 7a518a61e..26c8dcbda 100644 --- a/crates/switchyard-soak/tests/soak.rs +++ b/crates/switchyard-soak/tests/soak.rs @@ -9,7 +9,7 @@ use std::sync::Arc; use axum::Router; use axum::extract::{Json, State}; -use axum::http::{StatusCode, header}; +use axum::http::{HeaderMap, StatusCode, header}; use axum::response::{IntoResponse, Response}; use axum::routing::{get, post}; use clap::Parser; @@ -66,7 +66,11 @@ fn switchyard(is_chat_broken: bool) -> (Router, Arc>>) { async fn models() -> Response { Json(json!({"data": [{"id": "soak-route"}]})).into_response() } - async fn chat(State(state): State, Json(body): Json) -> Response { + async fn chat( + State(state): State, + headers: HeaderMap, + Json(body): Json, + ) -> Response { let invalid_messages = body.get("messages").is_some_and(|messages| { messages .as_array() @@ -79,6 +83,9 @@ fn switchyard(is_chat_broken: bool) -> (Router, Arc>>) { (StatusCode::BAD_REQUEST, "messages must not be empty").into_response() }; } + if !headers.contains_key("x-switchyard-session-id") { + return (StatusCode::BAD_REQUEST, "missing session id").into_response(); + } state.record("chat", &body); if state.is_chat_broken { return if body.get("stream") == Some(&Value::Bool(true)) { @@ -89,11 +96,25 @@ fn switchyard(is_chat_broken: bool) -> (Router, Arc>>) { } response(&body, "choices") } - async fn messages(State(state): State, Json(body): Json) -> Response { + async fn messages( + State(state): State, + headers: HeaderMap, + Json(body): Json, + ) -> Response { + if !headers.contains_key("x-switchyard-session-id") { + return (StatusCode::BAD_REQUEST, "missing session id").into_response(); + } state.record("messages", &body); response(&body, "content") } - async fn responses(State(state): State, Json(body): Json) -> Response { + async fn responses( + State(state): State, + headers: HeaderMap, + Json(body): Json, + ) -> Response { + if !headers.contains_key("x-switchyard-session-id") { + return (StatusCode::BAD_REQUEST, "missing session id").into_response(); + } state.record("responses", &body); response(&body, "output") } @@ -146,6 +167,8 @@ fn args(base_url: &str, results_dir: &str) -> Result { "0.1", "--invalid-canary-interval", "0.1", + "--scenario", + "short-interactive", "--results-dir", results_dir, ]) @@ -220,6 +243,70 @@ async fn bad_responses_and_canary_write_a_failing_summary() -> TestResult { Ok(()) } +#[tokio::test] +async fn nonbaseline_scenarios_all_cross_the_public_request_path() -> TestResult { + let (app, _) = switchyard(false); + let server = serve(app).await?; + let dir = tempfile::tempdir()?; + let results_dir = dir.path().join("scenario-results"); + let args = Args::try_parse_from([ + "switchyard-soak", + "--base-url", + &server.base_url, + "--model", + "soak-route", + "--duration", + "0.8s", + "--concurrency", + "4", + "--report-interval", + "0.2", + "--invalid-canary-interval", + "0", + "--scenario", + "long-context", + "--scenario", + "decode-heavy", + "--scenario", + "prefix-reuse", + "--scenario", + "mixed-traffic", + "--scenario", + "growing-conversation", + "--scenario", + "large-tool-catalog", + "--scenario", + "tool-call-burst", + "--scenario", + "stage-transitions", + "--scenario", + "classifier-mix", + "--results-dir", + results_dir.to_str().ok_or("non-utf8 results dir")?, + ])?; + + assert_eq!(run(args).await?, 0); + let summary: Value = + serde_json::from_str(&std::fs::read_to_string(results_dir.join("summary.json"))?)?; + let successes = summary["scenario_successes"] + .as_object() + .ok_or("scenario_successes was not an object")?; + for scenario in [ + "long-context", + "decode-heavy", + "prefix-reuse", + "mixed-traffic", + "growing-conversation", + "large-tool-catalog", + "tool-call-burst", + "stage-transitions", + "classifier-mix", + ] { + assert!(successes.contains_key(scenario), "missing {scenario}"); + } + Ok(()) +} + #[tokio::test] async fn unknown_model_is_rejected_before_the_run_starts() -> TestResult { let (app, _) = switchyard(false); diff --git a/docs/operations/soak_test.md b/docs/operations/soak_test.md index e5ec5bf6e..3c41d9753 100644 --- a/docs/operations/soak_test.md +++ b/docs/operations/soak_test.md @@ -5,18 +5,41 @@ server long enough to expose failures that short tests miss. Run it for 48 hours before code freeze when a release changes libsy, the Rust server, routing, streaming, translation, or server lifecycle behavior. -The test sends closed-loop traffic through: +The standard test sends closed-loop traffic through: - OpenAI Chat Completions (`/v1/chat/completions`) - Anthropic Messages (`/v1/messages`) - OpenAI Responses (`/v1/responses`) - streaming and non-streaming responses -- four repeated prompt prefixes +- short and long inputs, long outputs, shared prefixes, and mixed request sizes +- growing conversations, large tool catalogs, tool-call bursts, and stage transitions +- deterministic easy/hard classifier mixes The runner also checks `/health` and `/metrics` every minute. Every five minutes, it sends an invalid Chat Completions request, expects HTTP 400, and then confirms the server is still live. +The scenario catalog covers these distinct pressure angles: + +| Scenario | Pressure angle | What to review | +|---|---|---| +| `short-interactive` | Short prompts under fixed load, a concurrency knee, or a 10x request-rate burst | HTTP ceiling, TTFT, routing overhead, and the saturation point | +| `long-context` | 8K, 32K, and near-window inputs | TTFT, memory, and context-sensitive route failures | +| `decode-heavy` | 512-token and 1,024-token output limits | ITL, output tokens/second, and stream stability | +| `prefix-reuse` | Matched shared and unique long prefixes | Cache-sensitive TTFT and token accounting | +| `mixed-traffic` | A 70/20/10 short, medium, and long mix | p99 latency and head-of-line effects | +| `growing-conversation` | Eight cumulative conversation turns in one session | Affinity, history cost, and per-turn latency growth | +| `large-tool-catalog` | 16-tool and 64-tool JSON-schema catalogs | Serialization/routing overhead and intact tool forwarding | +| `tool-call-burst` | Eight linked assistant call/tool-result turns | Session continuity and burst handling | +| `stage-transitions` | One growing history across exploration, critical failure, and productive work | Stage-router tier changes and scorer output | +| `classifier-mix` | Deterministic 80/20 then 50/50 easy/hard requests | Target share, classifier calls/errors, and classifier latency | +| `context-overflow` | One target rejects a near-window request | Fallback to another eligible target | +| `failure-pressure` | Bounded 429, 500, malformed verdict, and truncated stream | Retry recovery, explicit terminal errors, and connection health | +| `client-cancellation` | A client timeout during a delayed response | Teardown and recovery of later traffic | + +`standard` includes the core and agentic rows. Run `resilience` separately because expected +failures should not be compared as throughput samples. + ## Prepare the server Run the exact commit, build, server config, backend, and model planned for the @@ -66,30 +89,56 @@ python3.12 scripts/benchmark_routing_algorithms.py \ --model stage_router=switchyard/stage \ --concurrency 100 \ --request-count 1000 \ + --scenario-set standard \ + --load-profile fixed \ + --profile-runs 3 \ --backend-label "release model deployment" ``` -oha sends non-streaming fixed-body requests and reports the raw HTTP request rate and latency -ceiling. AIPerf sends streaming Chat Completions and reports request latency, time to first token -(TTFT), inter-token latency (ITL), request throughput, and output-token throughput. The command -runs them one after the other because simultaneous runs would compete for the same server capacity -and distort both results. +The Rust crate exports one AIPerf `inputs-json` file per scenario. oha reuses the first exported +`short-interactive` payload as a non-streaming fixed body and reports the raw HTTP request rate and +latency ceiling. AIPerf replays every selected streaming session and reports request latency, time +to first token (TTFT), inter-token latency (ITL), request throughput, output-token throughput, and +multi-run confidence intervals. The command saves `/v1/stats` before and after each AIPerf run so +the report also records selected-target calls and shares, classifier calls/errors and latency, and +mean routing overhead. It runs +jobs sequentially because simultaneous tools would compete for the same server capacity. + +Use the load schedules after the fixed scenario comparison establishes a baseline: + +```bash +python3.12 scripts/benchmark_routing_algorithms.py \ + --base-url http://127.0.0.1:4000 \ + --model random=switchyard/random \ + --model classifier=switchyard/classifier \ + --scenario short-interactive \ + --load-profile concurrency-knee \ + --load-profile traffic-burst \ + --concurrency 128 \ + --request-rate 20 +``` + +The concurrency knee uses bounded steps up to `--concurrency`. The traffic burst holds the base +request rate, raises it to 10 times that rate for five seconds, then returns to the base. Pass +`--load-profile all` to run fixed, knee, and burst schedules. These schedules apply to the short +baseline; they are not separate request scenarios. To isolate routing overhead, configure every route to use the same target deployment and keep the -prompt, input length, output length, concurrency, and request count fixed. Set `--tokenizer` to the -real model tokenizer when exact token counts matter. AIPerf uses a deterministic workload seed so -each route receives the same synthetic inputs. +scenario manifest, concurrency, request count, and profile-run count fixed. Set `--tokenizer` to +the real model tokenizer when exact token counts matter. AIPerf uses deterministic sessions and a +fixed seed so each route receives the same requests. -This comparison command does not start VidaiMock or Switchyard. Point it at a running Switchyard -server backed by real models to measure end-to-end TTFT and token throughput. Real-model runs cost -tokens and include provider queuing and model variance, so use a dedicated deployment and repeat the -run before treating a small difference as an algorithm effect. +This comparison command does not start a backend or Switchyard. Point it at a running Switchyard +server backed by real models to measure end-to-end TTFT and token throughput with real tokenization +and generated tokens. Real-model runs cost tokens and include provider queuing and model variance, +so use a dedicated deployment and repeat the run before treating a small difference as an +algorithm effect. Use the local scenario backend first for deterministic routing correctness and +overhead, then rerun the same manifest against real models for capacity claims. ## Check the local server and load tools -The local test embeds [VidaiMock](https://github.com/vidaiUK/VidaiMock) as a Rust library, so it does -not need a separate `vidaimock` command. Build the server, soak tester, and mock helper from the -commit under test: +The local test uses a request-aware Axum backend from the soak crate. Build the server, soak tester, +and scenario backend from the commit under test: ```bash cargo build --release -p switchyard-server -p switchyard-soak \ @@ -120,30 +169,49 @@ measured requests each load tool sends to each algorithm. `--help` explains ever on `PATH` or not under `target/release`. The script checks all five commands before starting a process. A missing command prints a warning, -the build or install command, and the matching environment variable. Cargo compiles the embedded -VidaiMock library, but it does not install the unrelated oha or AIPerf programs during a normal -build. +the build or install command, and the matching environment variable. Cargo builds the scenario +backend, but it does not install oha or AIPerf during a normal build. The script gives each tool one job: | Tool | Job in the local test | |---|---| -| Embedded VidaiMock | Supplies local OpenAI-compatible model responses with configurable latency and no provider cost. | -| oha | Sends raw concurrent HTTP requests through every route and writes a status and latency distribution for each algorithm. | -| Python route checks | Sends one Chat Completions request through each route. The stage request includes a critical tool failure so the signal scorer selects the capable target. | -| AIPerf | Sends the same deterministic streaming workload through every route and records LLM, token, and response-time results. | -| Combined report | Joins selected oha and AIPerf metrics by algorithm in Markdown, CSV, and JSON. | -| `switchyard-soak` | Runs passthrough routing through Chat Completions, Messages, and Responses, with streaming on and off, while checking server health, metrics, process use, and required results. | +| Scenario backend | Returns local OpenAI-compatible responses, valid easy/hard classifier verdicts, and bounded failures with no provider cost. | +| oha | Measures the non-streaming `short-interactive` HTTP baseline through every route. | +| Python route checks | Sends one ordinary Chat Completions request through each configured route before load starts. | +| AIPerf | Replays Rust-exported streaming sessions through every route and records LLM, token, response-time, and confidence results. | +| Combined report | Joins scenario, load, oha, AIPerf, and routing-counter metrics in Markdown, CSV, and JSON. It keeps resilience rows separate from throughput rows. | +| `switchyard-soak` | Runs the standard scenario set while checking public API variants, server health, metrics, process use, and required results. | `scripts/local_soak_test.toml` exercises `noop`, `random`, `passthrough`, `llm_classifier`, and `stage_router`. It uses the accepted maximum retry count (10), a zero-weight random target, and the upper classifier and stage thresholds (1.0). Classifier affinity is disabled so every measured -request includes the classifier call. VidaiMock's generic classifier reply is intentionally not a -valid routing verdict, so the local classifier row measures the classifier call and fail-open path. -Use a real classifier backend or a purpose-built classifier stub to measure successful -classification. The config is validated with +request includes the classifier call. The scenario backend returns `p_solve=1.0` for easy markers +and `p_solve=0.1` for hard markers; with the configured threshold, those requests select the weak +and strong targets respectively. The config is validated with `switchyard-server --dry-run` before either service starts. +Run resilience cases separately so expected transport failures do not contaminate throughput +comparisons: + +```bash +python3.12 scripts/benchmark_routing_algorithms.py \ + --base-url http://127.0.0.1:4000 \ + --model classifier=switchyard/classifier \ + --scenario-set resilience \ + --load-profile fixed \ + --profile-runs 1 +``` + +`context-overflow` checks target fallback, `failure-pressure` injects bounded 429, 500, malformed +classifier, and truncated-stream cases, and `client-cancellation` uses a one-second client timeout +against a delayed response. Their expected error-rate ranges appear in the report's Resilience +section, and the command fails after writing the report when any row misses its range. The local +runner calls the scenario backend's `/reset` endpoint before each AIPerf cell so every algorithm +receives the same transient-failure sequence. When you run the comparison command directly against +that backend, pass `--scenario-backend-reset-url http://127.0.0.1:8100/reset` to preserve the same +comparison. + The runner does not add limits for target count or recent-turn history because the server does not limit them. Rust tests cover the real limits: 4,096 saved route assignments, 100,000 response-time samples, and 10,000 error records. Server tests cover invalid thresholds, bad classifier responses, diff --git a/scripts/benchmark_routing_algorithms.py b/scripts/benchmark_routing_algorithms.py index 646e42883..aaac6dab0 100755 --- a/scripts/benchmark_routing_algorithms.py +++ b/scripts/benchmark_routing_algorithms.py @@ -1,16 +1,18 @@ #!/usr/bin/env python3 # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Compare Switchyard routing algorithms with oha and NVIDIA AIPerf.""" +"""Compare Switchyard routes with Rust-owned scenarios, oha, and NVIDIA AIPerf.""" import argparse import csv import json +import math import os -import re import shutil import subprocess import sys +import urllib.error +import urllib.request from collections.abc import Sequence from dataclasses import asdict, dataclass from datetime import datetime, timezone @@ -39,27 +41,81 @@ class BenchmarkConfig: output_dir: Path oha_bin: str aiperf_bin: str + soak_bin: str tokenizer: str = "builtin" - input_sequence_length: int = 32 - output_sequence_length: int = 8 + scenario_set: str = "standard" + scenarios: tuple[str, ...] = () + load_profiles: tuple[str, ...] = ("fixed",) + profile_runs: int = 3 + prompt_bytes: int = 1024 + max_output_tokens: int = 32 + context_window_tokens: int = 32_768 + request_rate: float | None = None + scenario_backend_reset_url: str | None = None + + +@dataclass(frozen=True) +class ScenarioDefinition: + """One validated entry exported by switchyard-soak.""" + + id: str + group: str + description: str + expected: str + expected_error_rate_min: float + expected_error_rate_max: float + input_file: Path + load_profiles: tuple[dict[str, object], ...] + + +@dataclass(frozen=True) +class RoutingDelta: + """Routing counters attributable to one sequential AIPerf run.""" + + model_calls: str + model_share: str + model_errors: str + classifier_calls: int + classifier_errors: int + classifier_latency_avg_ms: float | None + routing_overhead_avg_ms: float | None @dataclass(frozen=True) class BenchmarkResult: - """Selected oha and AIPerf metrics for one routing algorithm.""" + """Selected load-tool and routing metrics for one comparison row.""" algorithm: str model: str - oha_requests_per_second: float - oha_latency_p50_ms: float - oha_latency_p99_ms: float - aiperf_requests_per_second: float - aiperf_request_latency_p50_ms: float - aiperf_ttft_p50_ms: float - aiperf_ttft_p99_ms: float + scenario: str + scenario_group: str + load_profile: str + expected_behavior: str + expected_error_rate_min: float + expected_error_rate_max: float + expectation_met: bool + oha_requests_per_second: float | None + oha_latency_p50_ms: float | None + oha_latency_p99_ms: float | None + aiperf_requests_per_second: float | None + aiperf_request_latency_p50_ms: float | None + aiperf_ttft_p50_ms: float | None + aiperf_ttft_p99_ms: float | None aiperf_itl_p50_ms: float | None - aiperf_output_tokens_per_second: float + aiperf_output_tokens_per_second: float | None aiperf_output_tokens_per_second_per_user_p50: float | None + aiperf_error_requests: float + aiperf_error_rate: float + aiperf_request_throughput_cv: float | None + aiperf_request_throughput_ci_low: float | None + aiperf_request_throughput_ci_high: float | None + selected_model_calls: str + selected_model_share: str + selected_model_errors: str + classifier_calls: int + classifier_errors: int + classifier_latency_avg_ms: float | None + routing_overhead_avg_ms: float | None def positive_int(value: str) -> int: @@ -70,24 +126,28 @@ def positive_int(value: str) -> int: return parsed +def positive_float(value: str) -> float: + """Parse a finite command-line number that must be greater than zero.""" + parsed = float(value) + if parsed <= 0 or not math.isfinite(parsed): + raise argparse.ArgumentTypeError("must be a finite number greater than zero") + return parsed + + def model_spec(value: str) -> tuple[str, str]: - """Parse LABEL=MODEL and reject labels that cannot be directory names.""" + """Parse LABEL=MODEL without coupling the report label to a filesystem name.""" label, separator, model = value.partition("=") - if not separator or not label or not model: + if not separator or not label.strip() or not model.strip(): raise argparse.ArgumentTypeError("must use LABEL=MODEL") - if re.fullmatch(r"[A-Za-z0-9_.-]+", label) is None: - raise argparse.ArgumentTypeError( - "LABEL may contain only letters, numbers, periods, underscores, and hyphens" - ) - return label, model + return label.strip(), model.strip() def parser() -> argparse.ArgumentParser: """Build the command-line interface.""" command = argparse.ArgumentParser( description=( - "Run sequential oha and AIPerf jobs for Switchyard models, then write one routing " - "algorithm performance report." + "Export Rust-owned request scenarios, run oha for the short baseline and AIPerf " + "for every selected scenario, then write one routing performance report." ), formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) @@ -99,56 +159,54 @@ def parser() -> argparse.ArgumentParser: type=model_spec, dest="models", metavar="LABEL=MODEL", - help="algorithm label and exact model id; repeat for every route to compare", - ) - command.add_argument( - "--concurrency", - type=positive_int, - default=4, - help="concurrent requests used by both load tools", + help="report label and exact model id; repeat for every route to compare", ) + command.add_argument("--concurrency", type=positive_int, default=4) + command.add_argument("--request-count", type=positive_int, default=100) + command.add_argument("--tokenizer", default="builtin") command.add_argument( - "--request-count", - type=positive_int, - default=100, - help="measured requests sent by each tool for each algorithm", + "--scenario-set", + choices=("core", "agentic", "resilience", "standard", "all"), + default="standard", ) command.add_argument( - "--tokenizer", - default="builtin", - help="AIPerf tokenizer; use the real model tokenizer for provider benchmarks", + "--scenario", + action="append", + default=[], + help="exact Rust scenario id; repeat to preserve an explicit order", ) command.add_argument( - "--input-sequence-length", - type=positive_int, - default=32, - help="synthetic AIPerf input token count", + "--load-profile", + action="append", + choices=("fixed", "concurrency-knee", "traffic-burst", "all"), + default=[], + help="load schedule to run; repeat as needed", ) command.add_argument( - "--output-sequence-length", + "--profile-runs", type=positive_int, - default=8, - help="requested AIPerf output token count", - ) - command.add_argument( - "--backend-label", - default="unspecified backend", - help="backend description recorded in the generated report", + default=3, + help="AIPerf repetitions used for confidence intervals; maximum 10", ) command.add_argument( - "--output-dir", - type=Path, - help="new directory for logs, raw tool results, and the combined report", + "--request-rate", + type=positive_float, + help="base requests/second for traffic-burst; defaults to concurrency", ) + command.add_argument("--prompt-bytes", type=positive_int, default=1024) + command.add_argument("--max-output-tokens", type=positive_int, default=32) + command.add_argument("--context-window-tokens", type=positive_int, default=32_768) + command.add_argument("--backend-label", default="unspecified backend") command.add_argument( - "--oha-bin", - default=os.environ.get("OHA_BIN", "oha"), - help="oha executable path", + "--scenario-backend-reset-url", + help="optional local scenario-backend endpoint reset before each AIPerf cell", ) + command.add_argument("--output-dir", type=Path) + command.add_argument("--oha-bin", default=os.environ.get("OHA_BIN", "oha")) + command.add_argument("--aiperf-bin", default=os.environ.get("AIPERF_BIN", "aiperf")) command.add_argument( - "--aiperf-bin", - default=os.environ.get("AIPERF_BIN", "aiperf"), - help="AIPerf executable path", + "--soak-bin", + default=os.environ.get("SWITCHYARD_SOAK_BIN", "target/release/switchyard-soak"), ) return command @@ -176,7 +234,6 @@ def resolve_binaries(requirements: Sequence[RequiredBinary]) -> dict[str, str]: missing.append(requirement) else: resolved[requirement.label] = path - for requirement in missing: print( f"warning: {requirement.label} executable not found: {requirement.value}", @@ -218,81 +275,397 @@ def read_json_object(path: Path) -> dict[str, object]: return document -def number_at(document: dict[str, object], group: str, field: str, path: Path) -> float: - """Read one required numeric metric from a nested result object.""" - metrics = document.get(group) - value = metrics.get(field) if isinstance(metrics, dict) else None - if not isinstance(value, int | float): - raise RuntimeError(f"missing numeric {group}.{field} in {path}") - return float(value) +def capture_stats(base_url: str, path: Path) -> dict[str, object]: + """Capture Switchyard's routing counters before or after one isolated run.""" + try: + with urllib.request.urlopen(f"{base_url.rstrip('/')}/v1/stats", timeout=10) as response: + document = json.loads(response.read()) + except (OSError, urllib.error.URLError, json.JSONDecodeError) as error: + raise RuntimeError(f"could not read Switchyard stats: {error}") from error + if not isinstance(document, dict): + raise RuntimeError("Switchyard /v1/stats did not return a JSON object") + path.write_text(f"{json.dumps(document, indent=2)}\n", encoding="utf-8") + return document -def optional_number_at(document: dict[str, object], group: str, field: str) -> float | None: - """Read one metric that AIPerf may omit for a one-token response.""" - metrics = document.get(group) - value = metrics.get(field) if isinstance(metrics, dict) else None - return float(value) if isinstance(value, int | float) else None +def reset_scenario_backend(url: str | None) -> None: + """Reset local failure injection so every algorithm receives the same attempts.""" + if url is None: + return + request = urllib.request.Request(url, data=b"", method="POST") + try: + with urllib.request.urlopen(request, timeout=10) as response: + document = json.loads(response.read()) + except (OSError, urllib.error.URLError, json.JSONDecodeError) as error: + raise RuntimeError(f"could not reset the scenario backend: {error}") from error + status = document.get("status") if isinstance(document, dict) else None + if response.status != 200 or status != "reset": + raise RuntimeError(f"scenario backend reset returned an invalid response from {url}") -def milliseconds(value: float, unit: object, metric: str, path: Path) -> float: - """Convert an AIPerf latency metric to milliseconds.""" - if unit == "ms": - return value - if unit in {"s", "seconds"}: - return value * 1000 - raise RuntimeError(f"unsupported {metric} unit {unit!r} in {path}") +def _nested_number(document: dict[str, object], *keys: str) -> float: + current: object = document + for key in keys: + current = current.get(key) if isinstance(current, dict) else None + return float(current) if isinstance(current, int | float) else 0.0 -def aiperf_latency(document: dict[str, object], metric: str, field: str, path: Path) -> float: - """Read one required AIPerf latency statistic in milliseconds.""" - value = number_at(document, metric, field, path) - group = document.get(metric) - unit = group.get("unit") if isinstance(group, dict) else None - return milliseconds(value, unit, metric, path) +def _histogram_delta( + before: dict[str, object], after: dict[str, object], *keys: str +) -> float | None: + count = _nested_number(after, *keys, "count") - _nested_number(before, *keys, "count") + total = _nested_number(after, *keys, "total_ms") - _nested_number(before, *keys, "total_ms") + return total / count if count > 0 else None -def optional_aiperf_latency( - document: dict[str, object], metric: str, field: str, path: Path -) -> float | None: - """Read one optional AIPerf latency statistic in milliseconds.""" - value = optional_number_at(document, metric, field) +def stats_delta(before: dict[str, object], after: dict[str, object]) -> RoutingDelta: + """Return routing and target-counter changes for one run.""" + before_value = before.get("models") + after_value = after.get("models") + before_models: dict[str, object] = before_value if isinstance(before_value, dict) else {} + after_models: dict[str, object] = after_value if isinstance(after_value, dict) else {} + classifier_before = before.get("classifier") + classifier_after = after.get("classifier") + before_classifier_models = ( + classifier_before.get("models") if isinstance(classifier_before, dict) else None + ) + after_classifier_models = ( + classifier_after.get("models") if isinstance(classifier_after, dict) else None + ) + calls = {} + errors = {} + for model, current in after_models.items(): + if not isinstance(model, str) or not isinstance(current, dict): + continue + previous = before_models.get(model) + previous_calls = previous.get("calls", 0) if isinstance(previous, dict) else 0 + classifier_current = ( + after_classifier_models.get(model) + if isinstance(after_classifier_models, dict) + else None + ) + classifier_previous = ( + before_classifier_models.get(model) + if isinstance(before_classifier_models, dict) + else None + ) + classifier_call_delta = float( + classifier_current.get("calls", 0) if isinstance(classifier_current, dict) else 0 + ) - float( + classifier_previous.get("calls", 0) if isinstance(classifier_previous, dict) else 0 + ) + call_delta = int( + float(current.get("calls", 0)) - float(previous_calls) - classifier_call_delta + ) + if call_delta > 0: + calls[model] = call_delta + previous_errors = previous.get("errors", 0) if isinstance(previous, dict) else 0 + classifier_error_delta = float( + classifier_current.get("errors", 0) if isinstance(classifier_current, dict) else 0 + ) - float( + classifier_previous.get("errors", 0) if isinstance(classifier_previous, dict) else 0 + ) + error_delta = int( + float(current.get("errors", 0)) - float(previous_errors) - classifier_error_delta + ) + if error_delta > 0: + errors[model] = error_delta + + classifier_count = classifier_total = 0.0 + if isinstance(after_classifier_models, dict): + for model, current in after_classifier_models.items(): + if not isinstance(model, str) or not isinstance(current, dict): + continue + previous = ( + before_classifier_models.get(model) + if isinstance(before_classifier_models, dict) + else None + ) + before_record = previous if isinstance(previous, dict) else {} + classifier_count += _nested_number(current, "model_call_latency", "count") + classifier_count -= _nested_number(before_record, "model_call_latency", "count") + classifier_total += _nested_number(current, "model_call_latency", "total_ms") + classifier_total -= _nested_number(before_record, "model_call_latency", "total_ms") + + total_calls = sum(calls.values()) + shares = {model: round(call_count / total_calls, 4) for model, call_count in calls.items()} + return RoutingDelta( + model_calls=json.dumps(calls, sort_keys=True, separators=(",", ":")), + model_share=json.dumps(shares, sort_keys=True, separators=(",", ":")), + model_errors=json.dumps(errors, sort_keys=True, separators=(",", ":")), + classifier_calls=int( + _nested_number(after, "classifier", "total_requests") + - _nested_number(before, "classifier", "total_requests") + ), + classifier_errors=int( + _nested_number(after, "classifier", "total_errors") + - _nested_number(before, "classifier", "total_errors") + ), + classifier_latency_avg_ms=( + classifier_total / classifier_count if classifier_count > 0 else None + ), + routing_overhead_avg_ms=_histogram_delta(before, after, "routing_overhead"), + ) + + +def validate_load_profile(profile: dict[str, object], scenario_id: str) -> None: + """Reject malformed or unbounded schedules before invoking a load generator.""" + profile_id = profile.get("id") + kind = profile.get("kind") + if (profile_id, kind) == ("fixed", "fixed"): + return + if (profile_id, kind) == ("concurrency-knee", "concurrency_knee"): + steps = profile.get("concurrency_steps") + if ( + not isinstance(steps, list) + or not steps + or not all(isinstance(step, int) and 1 <= step <= 10_000 for step in steps) + ): + raise RuntimeError(f"invalid concurrency knee for scenario {scenario_id}") + return + if (profile_id, kind) == ("traffic-burst", "traffic_burst"): + duration = profile.get("duration_seconds") + points = profile.get("points") + if not isinstance(duration, int) or not 1 <= duration <= 3_600: + raise RuntimeError(f"invalid traffic-burst duration for scenario {scenario_id}") + if not isinstance(points, list) or not 2 <= len(points) <= 64: + raise RuntimeError(f"invalid traffic-burst points for scenario {scenario_id}") + previous = -1.0 + for point in points: + time_s = point.get("time_s") if isinstance(point, dict) else None + multiplier = point.get("rate_multiplier") if isinstance(point, dict) else None + if ( + not isinstance(time_s, int | float) + or not isinstance(multiplier, int | float) + or not math.isfinite(float(time_s)) + or not math.isfinite(float(multiplier)) + or float(time_s) <= previous + or float(time_s) > duration + or float(multiplier) <= 0 + ): + raise RuntimeError(f"invalid traffic-burst point for scenario {scenario_id}") + previous = float(time_s) + return + raise RuntimeError(f"unknown load profile for scenario {scenario_id}: {profile_id!r}") + + +def export_scenarios(config: BenchmarkConfig, index: int, model: str) -> list[ScenarioDefinition]: + """Ask the Rust crate for the only authoritative scenario manifest.""" + output_dir = config.output_dir / "scenario-definitions" / f"algorithm-{index:02d}" + command = [ + config.soak_bin, + "--model", + model, + "--scenario-set", + config.scenario_set, + "--prompt-bytes", + str(config.prompt_bytes), + "--max-output-tokens", + str(config.max_output_tokens), + "--context-window-tokens", + str(config.context_window_tokens), + "--export-scenarios", + str(output_dir), + ] + for scenario in config.scenarios: + command.extend(("--scenario", scenario)) + run_checked( + "scenario export", + command, + config.output_dir / f"scenario-export-{index:02d}.log", + ) + document = read_json_object(output_dir / "manifest.json") + if document.get("schema_version") != 1 or document.get("model") != model: + raise RuntimeError(f"unsupported scenario manifest in {output_dir}") + raw_scenarios = document.get("scenarios") + if not isinstance(raw_scenarios, list) or not raw_scenarios: + raise RuntimeError(f"scenario manifest has no scenarios: {output_dir}") + definitions = [] + seen = set() + root = output_dir.resolve() + for entry in raw_scenarios: + if not isinstance(entry, dict): + raise RuntimeError(f"invalid scenario entry in {output_dir}") + scenario_id = entry.get("id") + group = entry.get("group") + description = entry.get("description") + expected = entry.get("expected") + expected_error_rate = entry.get("expected_error_rate") + min_error_rate = ( + expected_error_rate.get("min_rate") if isinstance(expected_error_rate, dict) else None + ) + max_error_rate = ( + expected_error_rate.get("max_rate") if isinstance(expected_error_rate, dict) else None + ) + input_name = entry.get("input_file") + profiles = entry.get("load_profiles") + if ( + not isinstance(scenario_id, str) + or not scenario_id + or scenario_id in seen + or group not in {"core", "agentic", "resilience"} + or not isinstance(description, str) + or not description + or not isinstance(expected, str) + or not expected + or not isinstance(min_error_rate, int | float) + or not isinstance(max_error_rate, int | float) + or not 0 <= float(min_error_rate) <= float(max_error_rate) <= 1 + or not isinstance(input_name, str) + or not isinstance(profiles, list) + ): + raise RuntimeError(f"invalid scenario entry in {output_dir}") + input_file = (output_dir / input_name).resolve() + if not input_file.is_relative_to(root) or not input_file.is_file(): + raise RuntimeError(f"scenario input escapes or is missing from {output_dir}") + if not all( + isinstance(profile, dict) and isinstance(profile.get("id"), str) for profile in profiles + ): + raise RuntimeError(f"invalid load profile for scenario {scenario_id}") + for profile in profiles: + validate_load_profile(profile, scenario_id) + seen.add(scenario_id) + definitions.append( + ScenarioDefinition( + id=scenario_id, + group=group, + description=description, + expected=expected, + expected_error_rate_min=float(min_error_rate), + expected_error_rate_max=float(max_error_rate), + input_file=input_file, + load_profiles=tuple(profiles), + ) + ) + return definitions + + +def selected_profiles( + config: BenchmarkConfig, scenario: ScenarioDefinition +) -> list[dict[str, object]]: + """Select only schedules supported by this request shape.""" + requested = set(config.load_profiles) + if "all" in requested: + return list(scenario.load_profiles) + return [profile for profile in scenario.load_profiles if profile.get("id") in requested] + + +def _metric(document: dict[str, object], name: str, statistic: str) -> tuple[float | None, object]: + group = document.get(name) + if isinstance(group, dict): + value = group.get(statistic) + return (float(value) if isinstance(value, int | float) else None, group.get("unit")) + metrics = document.get("metrics") + aggregate = metrics.get(f"{name}_{statistic}") if isinstance(metrics, dict) else None + if isinstance(aggregate, dict): + value = aggregate.get("mean") + return ( + float(value) if isinstance(value, int | float) else None, + aggregate.get("unit"), + ) + return None, None + + +def _latency(document: dict[str, object], metric: str, statistic: str, path: Path) -> float | None: + value, unit = _metric(document, metric, statistic) if value is None: return None - group = document.get(metric) - unit = group.get("unit") if isinstance(group, dict) else None - return milliseconds(value, unit, metric, path) + if unit == "ms": + return value + if unit in {"s", "seconds"}: + return value * 1000 + raise RuntimeError(f"unsupported {metric} unit {unit!r} in {path}") -def parse_result(algorithm: str, model: str, oha_path: Path, aiperf_path: Path) -> BenchmarkResult: - """Combine the stable JSON summaries emitted by oha and AIPerf.""" - oha = read_json_object(oha_path) +def parse_result( + algorithm: str, + model: str, + scenario: ScenarioDefinition, + load_profile: str, + oha_path: Path | None, + aiperf_path: Path, + before_stats: dict[str, object], + after_stats: dict[str, object], +) -> BenchmarkResult: + """Combine one AIPerf run, an optional oha baseline, and routing counter deltas.""" + oha_rps = oha_p50 = oha_p99 = None + if oha_path is not None: + oha = read_json_object(oha_path) + success, _unit = _metric(oha, "summary", "successRate") + if success != 1: + raise RuntimeError(f"oha success rate for {model} was {success}; see {oha_path}") + oha_rps, _unit = _metric(oha, "summary", "requestsPerSec") + p50, _unit = _metric(oha, "latencyPercentiles", "p50") + p99, _unit = _metric(oha, "latencyPercentiles", "p99") + oha_p50 = p50 * 1000 if p50 is not None else None + oha_p99 = p99 * 1000 if p99 is not None else None aiperf = read_json_object(aiperf_path) - success_rate = number_at(oha, "summary", "successRate", oha_path) - if success_rate != 1: - raise RuntimeError(f"oha success rate for {model} was {success_rate:.2%}; see {oha_path}") - error_count = optional_number_at(aiperf, "error_request_count", "avg") - if error_count is not None and error_count != 0: - raise RuntimeError(f"AIPerf recorded {error_count:g} errors for {model}; see {aiperf_path}") - + errors, _unit = _metric(aiperf, "error_request_count", "avg") + error_count = errors or 0.0 + measured_requests, _unit = _metric(aiperf, "request_count", "avg") + error_rate = ( + error_count / measured_requests + if measured_requests is not None and measured_requests > 0 + else 1.0 + ) + expectation_met = ( + scenario.expected_error_rate_min <= error_rate <= scenario.expected_error_rate_max + ) + throughput, _unit = _metric(aiperf, "request_throughput", "avg") + output_throughput, _unit = _metric(aiperf, "output_token_throughput", "avg") + user_throughput, _unit = _metric(aiperf, "e2e_output_token_throughput", "p50") + aggregate = aiperf.get("metrics") + throughput_stats = ( + aggregate.get("request_throughput_avg") if isinstance(aggregate, dict) else None + ) + routing = stats_delta(before_stats, after_stats) return BenchmarkResult( algorithm=algorithm, model=model, - oha_requests_per_second=number_at(oha, "summary", "requestsPerSec", oha_path), - oha_latency_p50_ms=number_at(oha, "latencyPercentiles", "p50", oha_path) * 1000, - oha_latency_p99_ms=number_at(oha, "latencyPercentiles", "p99", oha_path) * 1000, - aiperf_requests_per_second=number_at(aiperf, "request_throughput", "avg", aiperf_path), - aiperf_request_latency_p50_ms=aiperf_latency(aiperf, "request_latency", "p50", aiperf_path), - aiperf_ttft_p50_ms=aiperf_latency(aiperf, "time_to_first_token", "p50", aiperf_path), - aiperf_ttft_p99_ms=aiperf_latency(aiperf, "time_to_first_token", "p99", aiperf_path), - aiperf_itl_p50_ms=optional_aiperf_latency( - aiperf, "inter_token_latency", "p50", aiperf_path + scenario=scenario.id, + scenario_group=scenario.group, + load_profile=load_profile, + expected_behavior=scenario.expected, + expected_error_rate_min=scenario.expected_error_rate_min, + expected_error_rate_max=scenario.expected_error_rate_max, + expectation_met=expectation_met, + oha_requests_per_second=oha_rps, + oha_latency_p50_ms=oha_p50, + oha_latency_p99_ms=oha_p99, + aiperf_requests_per_second=throughput, + aiperf_request_latency_p50_ms=_latency(aiperf, "request_latency", "p50", aiperf_path), + aiperf_ttft_p50_ms=_latency(aiperf, "time_to_first_token", "p50", aiperf_path), + aiperf_ttft_p99_ms=_latency(aiperf, "time_to_first_token", "p99", aiperf_path), + aiperf_itl_p50_ms=_latency(aiperf, "inter_token_latency", "p50", aiperf_path), + aiperf_output_tokens_per_second=output_throughput, + aiperf_output_tokens_per_second_per_user_p50=user_throughput, + aiperf_error_requests=error_count, + aiperf_error_rate=error_rate, + aiperf_request_throughput_cv=( + float(throughput_stats["cv"]) + if isinstance(throughput_stats, dict) + and isinstance(throughput_stats.get("cv"), int | float) + else None ), - aiperf_output_tokens_per_second=number_at( - aiperf, "output_token_throughput", "avg", aiperf_path + aiperf_request_throughput_ci_low=( + float(throughput_stats["ci_low"]) + if isinstance(throughput_stats, dict) + and isinstance(throughput_stats.get("ci_low"), int | float) + else None ), - aiperf_output_tokens_per_second_per_user_p50=optional_number_at( - aiperf, "output_token_throughput_per_user", "p50" + aiperf_request_throughput_ci_high=( + float(throughput_stats["ci_high"]) + if isinstance(throughput_stats, dict) + and isinstance(throughput_stats.get("ci_high"), int | float) + else None ), + selected_model_calls=routing.model_calls, + selected_model_share=routing.model_share, + selected_model_errors=routing.model_errors, + classifier_calls=routing.classifier_calls, + classifier_errors=routing.classifier_errors, + classifier_latency_avg_ms=routing.classifier_latency_avg_ms, + routing_overhead_avg_ms=routing.routing_overhead_avg_ms, ) @@ -303,25 +676,27 @@ def format_metric(value: float | None) -> str: def write_report(config: BenchmarkConfig, results: Sequence[BenchmarkResult]) -> None: """Write machine-readable and reviewer-readable comparison reports.""" + if not results: + raise RuntimeError("no scenario and load-profile combinations were selected") rows = [asdict(result) for result in results] payload = { "generated_at": datetime.now(timezone.utc).isoformat(), "backend": config.backend_label, "base_url": config.base_url, "concurrency": config.concurrency, - "request_count_per_tool_per_algorithm": config.request_count, + "request_count": config.request_count, + "profile_runs": config.profile_runs, + "scenario_set": config.scenario_set, + "selected_scenarios": list(config.scenarios), + "selected_load_profiles": list(config.load_profiles), "aiperf_tokenizer": config.tokenizer, - "aiperf_input_sequence_length": config.input_sequence_length, - "aiperf_output_sequence_length": config.output_sequence_length, "results": rows, } (config.output_dir / "report.json").write_text( f"{json.dumps(payload, indent=2)}\n", encoding="utf-8" ) - - fieldnames = list(rows[0]) with (config.output_dir / "report.csv").open("w", encoding="utf-8", newline="") as output: - writer = csv.DictWriter(output, fieldnames=fieldnames) + writer = csv.DictWriter(output, fieldnames=list(rows[0])) writer.writeheader() writer.writerows(rows) @@ -329,65 +704,111 @@ def write_report(config: BenchmarkConfig, results: Sequence[BenchmarkResult]) -> "# Routing algorithm performance", "", f"- Backend: {config.backend_label}", - f"- Concurrency: {config.concurrency}", - f"- Measured requests per tool per algorithm: {config.request_count}", - f"- AIPerf workload: streaming Chat Completions, ISL {config.input_sequence_length}, " - f"OSL {config.output_sequence_length}, tokenizer `{config.tokenizer}`", - "- AIPerf warmup: 1 second per algorithm", - "- Run order: all oha and AIPerf jobs ran sequentially", + f"- AIPerf repetitions per row: {config.profile_runs}", + f"- Scenario set: {config.scenario_set}", + "- Run order: algorithms ran back-to-back within each scenario and load; all jobs " + "remained sequential", "", - "| Algorithm | oha req/s | oha p50 ms | oha p99 ms | AIPerf req/s | " - "request p50 ms | TTFT p50 ms | TTFT p99 ms | ITL p50 ms | output tok/s | " - "output tok/s/user p50 |", - "|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|", + "## Throughput and latency", + "", + "| Algorithm | Scenario | Load | oha req/s | AIPerf req/s | request p50 ms | " + "TTFT p50 ms | TTFT p99 ms | ITL p50 ms | output tok/s | error rate | gate |", + "|---|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---|", ] for result in results: + if result.scenario_group == "resilience": + continue lines.append( - f"| {result.algorithm} | {format_metric(result.oha_requests_per_second)} | " - f"{format_metric(result.oha_latency_p50_ms)} | " - f"{format_metric(result.oha_latency_p99_ms)} | " + f"| {result.algorithm} | {result.scenario} | {result.load_profile} | " + f"{format_metric(result.oha_requests_per_second)} | " f"{format_metric(result.aiperf_requests_per_second)} | " f"{format_metric(result.aiperf_request_latency_p50_ms)} | " f"{format_metric(result.aiperf_ttft_p50_ms)} | " f"{format_metric(result.aiperf_ttft_p99_ms)} | " f"{format_metric(result.aiperf_itl_p50_ms)} | " f"{format_metric(result.aiperf_output_tokens_per_second)} | " - f"{format_metric(result.aiperf_output_tokens_per_second_per_user_p50)} |" + f"{result.aiperf_error_rate:.2%} | " + f"{'PASS' if result.expectation_met else 'FAIL'} |" ) lines.extend( ( "", - "oha uses non-streaming fixed-body requests to measure the HTTP request-rate and " - "full-response latency ceiling. AIPerf uses streaming requests to measure LLM-aware " - "latency and token throughput. Compare algorithms within one tool's columns; do not " - "compare oha values directly with AIPerf values.", + "## Routing behavior", "", - "A VidaiMock run isolates Switchyard routing and protocol overhead. It does not " - "predict production model capacity. Use the same command against routes backed by " - "the same real model deployment to measure end-to-end token performance.", + "| Algorithm | Scenario | Load | selected target calls | selected target share | target errors | " + "classifier calls | classifier errors | classifier avg ms | routing avg ms |", + "|---|---|---|---|---|---|---:|---:|---:|---:|", + ) + ) + for result in results: + lines.append( + f"| {result.algorithm} | {result.scenario} | {result.load_profile} | " + f"`{result.selected_model_calls}` | `{result.selected_model_share}` | " + f"`{result.selected_model_errors}` | " + f"{result.classifier_calls} | {result.classifier_errors} | " + f"{format_metric(result.classifier_latency_avg_ms)} | " + f"{format_metric(result.routing_overhead_avg_ms)} |" + ) + + resilience = [result for result in results if result.scenario_group == "resilience"] + if resilience: + lines.extend( + ( + "", + "## Resilience", + "", + "| Algorithm | Scenario | Load | client error rate | expected range | gate | " + "target errors | Expected behavior |", + "|---|---|---|---:|---|---|---|---|", + ) + ) + for result in resilience: + lines.append( + f"| {result.algorithm} | {result.scenario} | {result.load_profile} | " + f"{result.aiperf_error_rate:.2%} | " + f"{result.expected_error_rate_min:.2%}–{result.expected_error_rate_max:.2%} | " + f"{'PASS' if result.expectation_met else 'FAIL'} | " + f"`{result.selected_model_errors}` | " + f"{result.expected_behavior} |" + ) + lines.extend( + ( + "", + "oha runs only for the fixed short-interactive baseline. AIPerf replays the same " + "Rust-exported request payload for streaming, session-aware LLM metrics. Compare " + "algorithms within the same scenario and load row.", + "", + "The local scenario backend isolates Switchyard routing and protocol overhead. It " + "does not predict model capacity. Repeat the same command against routes backed by " + "one real deployment to include tokenization, model execution, and provider queuing.", "", ) ) (config.output_dir / "report.md").write_text("\n".join(lines), encoding="utf-8") -def run_oha(config: BenchmarkConfig, algorithm: str, model: str, output_dir: Path) -> Path: - """Run oha's fixed-body HTTP load for one algorithm.""" - request_path = output_dir / f"{algorithm}-request.json" - result_path = output_dir / f"{algorithm}.json" - request_path.write_text( - json.dumps( - { - "model": model, - "messages": [{"role": "user", "content": "Reply with exactly OK."}], - "max_tokens": config.output_sequence_length, - "stream": False, - } - ), - encoding="utf-8", - ) +def run_oha( + config: BenchmarkConfig, + label: str, + scenario: ScenarioDefinition, + output_dir: Path, +) -> Path: + """Run oha only for the fixed short-interactive baseline.""" + inputs = read_json_object(scenario.input_file) + data = inputs.get("data") + try: + payload = data[0]["payloads"][0] # type: ignore[index] + except (IndexError, KeyError, TypeError) as error: + raise RuntimeError(f"invalid AIPerf input file {scenario.input_file}") from error + if not isinstance(payload, dict): + raise RuntimeError(f"invalid AIPerf payload in {scenario.input_file}") + body = dict(payload) + body["stream"] = False + request_path = output_dir / f"{label}-request.json" + result_path = output_dir / f"{label}.json" + request_path.write_text(json.dumps(body), encoding="utf-8") run_checked( - f"oha {algorithm} load", + f"oha {label}", [ config.oha_bin, "-n", @@ -408,64 +829,93 @@ def run_oha(config: BenchmarkConfig, algorithm: str, model: str, output_dir: Pat str(result_path), f"{config.base_url.rstrip('/')}/v1/chat/completions", ], - output_dir / f"{algorithm}.log", + output_dir / f"{label}.log", ) return result_path def run_aiperf( config: BenchmarkConfig, - algorithm: str, + label: str, model: str, + scenario: ScenarioDefinition, + profile: dict[str, object], output_dir: Path, + concurrency: int | None = None, ) -> Path: - """Run one isolated AIPerf profile for an algorithm.""" - artifact_dir = output_dir / algorithm - run_checked( - f"AIPerf {algorithm} streaming profile", - [ - config.aiperf_bin, - "profile", - "--model", - model, - "--url", - config.base_url, - "--endpoint-type", - "chat", - "--streaming", - "--tokenizer", - config.tokenizer, - "--use-legacy-max-tokens", - "--isl", - str(config.input_sequence_length), - "--osl", - str(config.output_sequence_length), - "--random-seed", - "42", - "--warmup-duration", - "1", - "--concurrency", - str(config.concurrency), - "--request-count", - str(config.request_count), - "--failed-request-threshold", - "0", - "--request-timeout-seconds", - "30", - "--ui", - "none", - "--artifact-dir", - str(artifact_dir), - ], - output_dir / f"{algorithm}.log", - ) + """Run one isolated AIPerf profile using a Rust-exported inputs-json dataset.""" + artifact_dir = output_dir / label + command = [ + config.aiperf_bin, + "profile", + "--model", + model, + "--url", + config.base_url, + "--endpoint-type", + "chat", + "--streaming", + "--tokenizer", + config.tokenizer, + "--custom-dataset-type", + "inputs-json", + "--input-file", + str(scenario.input_file), + "--session-header", + "x-switchyard-session-id", + "--random-seed", + "42", + "--warmup-duration", + "1", + "--num-profile-runs", + str(config.profile_runs), + "--failed-request-threshold", + "1", + "--request-timeout-seconds", + "1" if scenario.id == "client-cancellation" else "30", + "--ui", + "none", + "--artifact-dir", + str(artifact_dir), + ] + kind = profile.get("kind") + if kind == "traffic_burst": + points = profile.get("points") + if not isinstance(points, list): + raise RuntimeError(f"traffic-burst profile has no points for {scenario.id}") + base_rate = config.request_rate or float(config.concurrency) + rate_points = [] + for point in points: + if not isinstance(point, dict): + raise RuntimeError(f"invalid traffic-burst point for {scenario.id}") + rate_points.append( + { + "time_s": point["time_s"], + "qps": base_rate * float(point["rate_multiplier"]), + } + ) + series_path = output_dir / f"{label}-request-rate.json" + series_path.write_text( + f"{json.dumps({'points': rate_points}, indent=2)}\n", encoding="utf-8" + ) + command.extend(("--request-rate-series", str(series_path), "--arrival-pattern", "constant")) + command.extend(("--benchmark-duration", str(profile.get("duration_seconds")))) + command.extend(("--concurrency", str(config.concurrency))) + else: + command.extend(("--concurrency", str(concurrency or config.concurrency))) + command.extend(("--request-count", str(config.request_count))) + run_checked(f"AIPerf {label}", command, output_dir / f"{label}.log") + if config.profile_runs > 1: + return artifact_dir / "aggregate" / "profile_export_aiperf_aggregate.json" return artifact_dir / "profile_export_aiperf.json" def run_benchmark(config: BenchmarkConfig) -> Path: - """Run both tools for every algorithm and return the report directory.""" + """Run selected scenarios for every algorithm and return the report directory.""" if not config.models: raise RuntimeError("at least one algorithm model is required") + if config.profile_runs > 10: + raise RuntimeError("--profile-runs must be between 1 and 10") labels = [label for label, _model in config.models] if len(labels) != len(set(labels)): raise RuntimeError("algorithm labels must be unique") @@ -474,19 +924,77 @@ def run_benchmark(config: BenchmarkConfig) -> Path: aiperf_dir = config.output_dir / "aiperf" oha_dir.mkdir() aiperf_dir.mkdir() - - oha_paths = {} - for algorithm, model in config.models: - oha_paths[algorithm] = run_oha(config, algorithm, model, oha_dir) - aiperf_paths = {} - for algorithm, model in config.models: - aiperf_paths[algorithm] = run_aiperf(config, algorithm, model, aiperf_dir) - results = [ - parse_result(algorithm, model, oha_paths[algorithm], aiperf_paths[algorithm]) - for algorithm, model in config.models + results = [] + definition_sets = [ + export_scenarios(config, index, model) + for index, (_algorithm, model) in enumerate(config.models) ] + expected_ids = [scenario.id for scenario in definition_sets[0]] + for definitions in definition_sets[1:]: + if [scenario.id for scenario in definitions] != expected_ids: + raise RuntimeError("scenario exports differ between algorithms") + for scenario_index, baseline_scenario in enumerate(definition_sets[0]): + for profile in selected_profiles(config, baseline_scenario): + profile_id = str(profile["id"]) + concurrencies: list[int | None] = [None] + if profile.get("kind") == "concurrency_knee": + steps = profile.get("concurrency_steps") + if not isinstance(steps, list) or not all(isinstance(step, int) for step in steps): + raise RuntimeError(f"invalid concurrency knee for {baseline_scenario.id}") + concurrencies = sorted( + set( + [step for step in steps if step <= config.concurrency] + + [config.concurrency] + ) + ) + for concurrency in concurrencies: + load_label = profile_id if concurrency is None else f"{profile_id}@{concurrency}" + for index, (algorithm, model) in enumerate(config.models): + scenario = definition_sets[index][scenario_index] + artifact_label = ( + f"algorithm-{index:02d}-{scenario.id}-{load_label.replace('@', '-')}" + ) + oha_path = None + if scenario.id == "short-interactive" and profile_id == "fixed": + oha_path = run_oha(config, artifact_label, scenario, oha_dir) + reset_scenario_backend(config.scenario_backend_reset_url) + before = capture_stats( + config.base_url, + aiperf_dir / f"{artifact_label}-stats-before.json", + ) + aiperf_path = run_aiperf( + config, + artifact_label, + model, + scenario, + profile, + aiperf_dir, + concurrency, + ) + after = capture_stats( + config.base_url, + aiperf_dir / f"{artifact_label}-stats-after.json", + ) + results.append( + parse_result( + algorithm, + model, + scenario, + load_label, + oha_path, + aiperf_path, + before, + after, + ) + ) write_report(config, results) + failures = [result for result in results if not result.expectation_met] + if failures: + raise RuntimeError( + f"{len(failures)} benchmark row(s) missed their error-rate gate; " + f"see {config.output_dir / 'report.md'}" + ) return config.output_dir @@ -512,6 +1020,12 @@ def main(argv: Sequence[str] | None = None) -> int: args.aiperf_bin, "Install AIPerf with: uv tool install --python 3.12 aiperf", ), + RequiredBinary( + "switchyard-soak", + "SWITCHYARD_SOAK_BIN", + args.soak_bin, + "Build the scenario exporter with: cargo build --release -p switchyard-soak", + ), ) ) output_dir = run_benchmark( @@ -521,12 +1035,20 @@ def main(argv: Sequence[str] | None = None) -> int: concurrency=args.concurrency, request_count=args.request_count, tokenizer=args.tokenizer, - input_sequence_length=args.input_sequence_length, - output_sequence_length=args.output_sequence_length, + scenario_set=args.scenario_set, + scenarios=tuple(args.scenario), + load_profiles=tuple(args.load_profile or ["fixed"]), + profile_runs=args.profile_runs, + prompt_bytes=args.prompt_bytes, + max_output_tokens=args.max_output_tokens, + context_window_tokens=args.context_window_tokens, + request_rate=args.request_rate, backend_label=args.backend_label, output_dir=(args.output_dir or default_output_dir()).resolve(), oha_bin=binaries["oha"], aiperf_bin=binaries["AIPerf"], + soak_bin=binaries["switchyard-soak"], + scenario_backend_reset_url=args.scenario_backend_reset_url, ) ) except (OSError, RuntimeError) as error: diff --git a/scripts/run_local_soak_test.py b/scripts/run_local_soak_test.py index 342105c87..2d768a45d 100755 --- a/scripts/run_local_soak_test.py +++ b/scripts/run_local_soak_test.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Run every Switchyard route with VidaiMock, load tools, and the soak test.""" +"""Run every Switchyard route with the local scenario backend and load tools.""" import argparse import json @@ -67,8 +67,8 @@ def parser() -> argparse.ArgumentParser: """Build the plain-English command-line interface.""" command = argparse.ArgumentParser( description=( - "Start VidaiMock and a local Switchyard server, send one request through every route, " - "then run oha, AIPerf, and switchyard-soak." + "Start the request-aware scenario backend and a local Switchyard server, send one " + "request through every route, then run oha, AIPerf, and switchyard-soak." ), formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) @@ -93,7 +93,7 @@ def parser() -> argparse.ArgumentParser: "--mock-latency-ms", type=nonnegative_int, default=40, - help="response latency VidaiMock adds to each backend call", + help="response latency the local scenario backend adds to ordinary calls", ) command.add_argument( "--server-port", @@ -162,31 +162,12 @@ def wait_for_health(child: Child, url: str, expected_status: str | None = None) def check_routes(base_url: str, output_path: Path) -> None: """Send one HTTP request through every configured route.""" - ordinary = [{"role": "user", "content": "Reply with exactly OK."}] - stage_edge = [ - {"role": "user", "content": "Fix the build."}, - { - "role": "assistant", - "tool_calls": [ - { - "id": "call_1", - "type": "function", - "function": {"name": "Bash", "arguments": '{"command":"cargo test"}'}, - } - ], - }, - { - "role": "tool", - "tool_call_id": "call_1", - "content": "fatal runtime error: out of memory", - }, - ] records = [] - for algorithm, route in ROUTES: + for _algorithm, route in ROUTES: body = json.dumps( { "model": route, - "messages": stage_edge if algorithm == "stage_router" else ordinary, + "messages": [{"role": "user", "content": "Reply with exactly OK."}], "max_tokens": 8, "stream": False, } @@ -244,7 +225,7 @@ def run_local_soak_test(args: argparse.Namespace) -> Path: rust_build, ), RequiredBinary( - "embedded VidaiMock helper", + "scenario backend", "SWITCHYARD_SOAK_MOCK_BIN", os.environ.get( "SWITCHYARD_SOAK_MOCK_BIN", @@ -268,7 +249,7 @@ def run_local_soak_test(args: argparse.Namespace) -> Path: ) server_bin = binaries["switchyard-server"] soak_bin = binaries["switchyard-soak"] - mock_bin = binaries["embedded VidaiMock helper"] + mock_bin = binaries["scenario backend"] oha_bin = binaries["oha"] aiperf_bin = binaries["AIPerf"] @@ -286,7 +267,7 @@ def run_local_soak_test(args: argparse.Namespace) -> Path: children: list[Child] = [] try: mock = start_child( - "embedded VidaiMock", + "scenario backend", [ mock_bin, "--port", @@ -294,7 +275,7 @@ def run_local_soak_test(args: argparse.Namespace) -> Path: "--latency-ms", str(args.mock_latency_ms), ], - output_dir / "vidaimock.log", + output_dir / "scenario-backend.log", ) children.append(mock) wait_for_health(mock, f"http://127.0.0.1:{MOCK_PORT}/health") @@ -315,10 +296,15 @@ def run_local_soak_test(args: argparse.Namespace) -> Path: models=ROUTES, concurrency=args.concurrency, request_count=args.request_count, - backend_label=f"VidaiMock with {args.mock_latency_ms} ms response latency", + backend_label=( + f"request-aware local backend with {args.mock_latency_ms} ms response latency" + ), output_dir=output_dir / "routing-benchmark", oha_bin=oha_bin, aiperf_bin=aiperf_bin, + soak_bin=soak_bin, + profile_runs=1, + scenario_backend_reset_url=f"http://127.0.0.1:{MOCK_PORT}/reset", ) ) diff --git a/tests/test_routing_performance_report.py b/tests/test_routing_performance_report.py index 4260271e7..e943a38b8 100644 --- a/tests/test_routing_performance_report.py +++ b/tests/test_routing_performance_report.py @@ -1,14 +1,21 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +import csv import json -from scripts.benchmark_routing_algorithms import BenchmarkConfig, parse_result, write_report +from scripts.benchmark_routing_algorithms import ( + BenchmarkConfig, + ScenarioDefinition, + parse_result, + write_report, +) -def test_report_combines_oha_and_aiperf_metrics(tmp_path) -> None: +def test_report_combines_scenarios_single_and_aggregate_results(tmp_path) -> None: oha_path = tmp_path / "oha.json" - aiperf_path = tmp_path / "profile_export_aiperf.json" + single_path = tmp_path / "profile_export_aiperf.json" + aggregate_path = tmp_path / "profile_export_aiperf_aggregate.json" oha_path.write_text( json.dumps( { @@ -17,18 +24,125 @@ def test_report_combines_oha_and_aiperf_metrics(tmp_path) -> None: } ) ) - aiperf_path.write_text( + single_path.write_text( json.dumps( { "error_request_count": {"unit": "requests", "avg": 0}, + "request_count": {"unit": "requests", "avg": 4}, "request_throughput": {"unit": "requests/sec", "avg": 95.5}, "request_latency": {"unit": "ms", "p50": 40.0}, "time_to_first_token": {"unit": "ms", "p50": 15.0, "p99": 35.0}, "output_token_throughput": {"unit": "tokens/sec", "avg": 620.0}, + "e2e_output_token_throughput": { + "unit": "tokens/sec/user", + "p50": 200.0, + }, } ) ) - result = parse_result("random", "switchyard/random", oha_path, aiperf_path) + aggregate_path.write_text( + json.dumps( + { + "metrics": { + "error_request_count_avg": {"unit": "requests", "mean": 2}, + "request_count_avg": {"unit": "requests", "mean": 4}, + "request_throughput_avg": { + "unit": "requests/sec", + "mean": 80.0, + "cv": 0.03, + "ci_low": 77.0, + "ci_high": 83.0, + }, + "request_latency_p50": {"unit": "seconds", "mean": 0.05}, + "time_to_first_token_p50": {"unit": "ms", "mean": 20.0}, + "time_to_first_token_p99": {"unit": "ms", "mean": 45.0}, + "output_token_throughput_avg": { + "unit": "tokens/sec", + "mean": 500.0, + }, + } + } + ) + ) + short = ScenarioDefinition( + id="short-interactive", + group="core", + description="short baseline", + expected="all requests succeed", + expected_error_rate_min=0.0, + expected_error_rate_max=0.0, + input_file=tmp_path / "short.json", + load_profiles=(), + ) + failure = ScenarioDefinition( + id="failure-pressure", + group="resilience", + description="failure injection", + expected="failures stay bounded", + expected_error_rate_min=0.01, + expected_error_rate_max=0.75, + input_file=tmp_path / "failure.json", + load_profiles=(), + ) + before = { + "models": { + "mock/weak": {"calls": 10, "errors": 0}, + "mock/classifier": {"calls": 3, "errors": 0}, + }, + "classifier": { + "total_requests": 3, + "total_errors": 0, + "models": { + "mock/classifier": { + "calls": 3, + "errors": 0, + "model_call_latency": {"count": 3, "total_ms": 6}, + } + }, + }, + "routing_overhead": {"count": 10, "total_ms": 20}, + } + after = { + "models": { + "mock/weak": {"calls": 20, "errors": 2}, + "mock/strong": {"calls": 2, "errors": 0}, + "mock/classifier": {"calls": 5, "errors": 1}, + }, + "classifier": { + "total_requests": 5, + "total_errors": 1, + "models": { + "mock/classifier": { + "calls": 5, + "errors": 1, + "model_call_latency": {"count": 5, "total_ms": 14}, + } + }, + }, + "routing_overhead": {"count": 22, "total_ms": 50}, + } + results = ( + parse_result( + "random", + "switchyard/random", + short, + "fixed", + oha_path, + single_path, + before, + after, + ), + parse_result( + "classifier", + "switchyard/classifier", + failure, + "fixed", + None, + aggregate_path, + before, + after, + ), + ) output_dir = tmp_path / "report" output_dir.mkdir() config = BenchmarkConfig( @@ -36,34 +150,29 @@ def test_report_combines_oha_and_aiperf_metrics(tmp_path) -> None: models=(("random", "switchyard/random"),), concurrency=100, request_count=1000, - tokenizer="builtin", - input_sequence_length=32, - output_sequence_length=8, backend_label="test backend", output_dir=output_dir, oha_bin="oha", aiperf_bin="aiperf", + soak_bin="switchyard-soak", ) - write_report(config, (result,)) + write_report(config, results) report = (output_dir / "report.md").read_text() - assert "| random | 120.50 | 12.00 | 45.00 | 95.50 |" in report - assert "| 40.00 | 15.00 | 35.00 | n/a | 620.00 | n/a |" in report - assert (output_dir / "report.csv").is_file() - assert json.loads((output_dir / "report.json").read_text())["results"] == [ - { - "algorithm": "random", - "model": "switchyard/random", - "oha_requests_per_second": 120.5, - "oha_latency_p50_ms": 12.0, - "oha_latency_p99_ms": 45.0, - "aiperf_requests_per_second": 95.5, - "aiperf_request_latency_p50_ms": 40.0, - "aiperf_ttft_p50_ms": 15.0, - "aiperf_ttft_p99_ms": 35.0, - "aiperf_itl_p50_ms": None, - "aiperf_output_tokens_per_second": 620.0, - "aiperf_output_tokens_per_second_per_user_p50": None, - } - ] + assert "| random | short-interactive | fixed | 120.50 | 95.50 |" in report + assert "## Resilience" in report + assert ( + "| classifier | failure-pressure | fixed | 50.00% | 1.00%–75.00% | PASS | `{" + '"mock/weak":2}` | failures stay bounded |' in report + ) + rows = list(csv.DictReader((output_dir / "report.csv").open())) + assert rows[0]["selected_model_calls"] == '{"mock/strong":2,"mock/weak":10}' + assert rows[0]["selected_model_share"] == '{"mock/strong":0.1667,"mock/weak":0.8333}' + assert rows[0]["selected_model_errors"] == '{"mock/weak":2}' + payload = json.loads((output_dir / "report.json").read_text()) + assert payload["results"][0]["aiperf_itl_p50_ms"] is None + assert payload["results"][0]["aiperf_output_tokens_per_second_per_user_p50"] == 200.0 + assert payload["results"][1]["aiperf_request_latency_p50_ms"] == 50.0 + assert payload["results"][1]["aiperf_request_throughput_cv"] == 0.03 + assert payload["results"][1]["classifier_latency_avg_ms"] == 4.0 From 3f827cced2d0a1b1ffd2affe98885230b092726e Mon Sep 17 00:00:00 2001 From: Elyas Mehtabuddin Date: Wed, 19 Aug 2026 09:46:28 -0700 Subject: [PATCH 8/9] feat(operations): report realistic routing overhead Signed-off-by: Elyas Mehtabuddin --- crates/switchyard-soak/README.md | 9 +- .../switchyard-soak/examples/mock_server.rs | 126 +++- docs/operations/soak_test.md | 39 +- scripts/aiperf_runner.py | 220 +++++++ scripts/benchmark_routing_algorithms.py | 605 +++++++++++++++--- scripts/run_local_soak_test.py | 21 +- tests/test_aiperf_runner.py | 157 +++++ tests/test_routing_performance_report.py | 265 +++++++- 8 files changed, 1285 insertions(+), 157 deletions(-) create mode 100644 scripts/aiperf_runner.py create mode 100644 tests/test_aiperf_runner.py diff --git a/crates/switchyard-soak/README.md b/crates/switchyard-soak/README.md index d6be78844..f49ffa379 100644 --- a/crates/switchyard-soak/README.md +++ b/crates/switchyard-soak/README.md @@ -92,14 +92,17 @@ The catalog contains: NVIDIA AIPerf sequentially for every routing algorithm. The backend returns valid classifier verdicts and injects the context, retry, stream, and cancellation cases named by resilience scenarios. It resets transient-failure counters before each AIPerf cell so every routing algorithm -receives the same attempts. The local test needs no provider key and incurs no inference cost. +receives the same attempts. For streaming requests it emits the scenario's requested output length +with a configurable delay between tokens, which keeps local TTFT, ITL, and token-throughput +comparisons deterministic. The local test needs no provider key and incurs no inference cost. Run `scripts/benchmark_routing_algorithms.py` by itself against an existing Switchyard server to compare the same routes with real model output. oha runs only for the fixed `short-interactive` baseline. AIPerf replays every selected session and reports TTFT, ITL, request throughput, output token throughput, multi-run confidence, selected-target calls, classifier calls, and routing -overhead. The operations guide explains how to keep the comparison fair and when to use the local -backend or real models. +overhead. Add `--direct-base-url` and `--direct-model` to compare each route with the same backend +without Switchyard. The operations guide explains how to keep the comparison fair and when to use +the local backend or real models. The script needs the built Rust programs plus installed `oha` and AIPerf commands. `switchyard-soak` itself does not require those extra programs and can run by itself against a diff --git a/crates/switchyard-soak/examples/mock_server.rs b/crates/switchyard-soak/examples/mock_server.rs index 50a7ffaa3..c92578fd6 100644 --- a/crates/switchyard-soak/examples/mock_server.rs +++ b/crates/switchyard-soak/examples/mock_server.rs @@ -4,16 +4,19 @@ //! Request-aware local backend used by `scripts/run_local_soak_test.py`. use std::collections::HashMap; +use std::convert::Infallible; use std::process::ExitCode; use std::sync::Arc; use std::time::Duration; use axum::Router; use axum::extract::{Json, State}; -use axum::http::{StatusCode, header}; +use axum::http::StatusCode; +use axum::response::sse::{Event, Sse}; use axum::response::{IntoResponse, Response}; use axum::routing::{get, post}; use clap::Parser; +use futures_util::{StreamExt, stream}; use parking_lot::Mutex; use serde_json::{Value, json}; @@ -32,14 +35,22 @@ struct Args { /// Artificial delay, in milliseconds, added to ordinary responses. #[arg(long, default_value_t = 40)] latency_ms: u64, + + /// Artificial delay between streamed output tokens, in milliseconds. + #[arg(long, default_value_t = 1)] + token_latency_ms: u64, } #[derive(Clone)] struct BackendState { latency: Duration, + token_latency: Duration, attempts: Arc>>, } +// AIPerf requires this OpenAI field; a fixed value keeps local runs deterministic. +const FIXED_CREATED_AT: u64 = 1_700_000_000; + fn scenario_marker(body: &Value) -> Option<&str> { body.get("messages")? .as_array()? @@ -65,6 +76,7 @@ fn completion(model: &str, content: &str) -> Value { json!({ "id": "chatcmpl-switchyard-soak", "object": "chat.completion", + "created": FIXED_CREATED_AT, "model": model, "choices": [{ "index": 0, @@ -75,30 +87,69 @@ fn completion(model: &str, content: &str) -> Value { }) } -fn stream(model: &str, truncated: bool) -> Response { - let first = json!({ - "id": "chatcmpl-switchyard-soak", - "object": "chat.completion.chunk", - "model": model, - "choices": [{ - "index": 0, - "delta": {"role": "assistant", "content": "OK"}, - "finish_reason": null - }] - }); - let end = json!({ - "id": "chatcmpl-switchyard-soak", - "object": "chat.completion.chunk", - "model": model, - "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], - "usage": {"prompt_tokens": 32, "completion_tokens": 2, "total_tokens": 34} - }); - let body = if truncated { - format!("data: {first}\n\n") - } else { - format!("data: {first}\n\ndata: {end}\n\ndata: [DONE]\n\n") - }; - ([(header::CONTENT_TYPE, "text/event-stream")], body).into_response() +fn requested_output_tokens(body: &Value) -> u64 { + body.get("max_completion_tokens") + .or_else(|| body.get("max_tokens")) + .and_then(Value::as_u64) + .unwrap_or(2) + .clamp(1, 4_096) +} + +fn stream( + model: &str, + truncated: bool, + completion_tokens: u64, + token_latency: Duration, +) -> Response { + let mut events = Vec::with_capacity(completion_tokens as usize + 2); + for index in 0..completion_tokens { + events.push( + json!({ + "id": "chatcmpl-switchyard-soak", + "object": "chat.completion.chunk", + "created": FIXED_CREATED_AT, + "model": model, + "choices": [{ + "index": 0, + "delta": { + "role": (index == 0).then_some("assistant"), + "content": if index == 0 { "token" } else { " token" } + }, + "finish_reason": null + }] + }) + .to_string(), + ); + if truncated { + break; + } + } + if !truncated { + events.push( + json!({ + "id": "chatcmpl-switchyard-soak", + "object": "chat.completion.chunk", + "created": FIXED_CREATED_AT, + "model": model, + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], + "usage": { + "prompt_tokens": 32, + "completion_tokens": completion_tokens, + "total_tokens": 32 + completion_tokens + } + }) + .to_string(), + ); + events.push("[DONE]".to_string()); + } + let events = + stream::iter(events.into_iter().enumerate()).then(move |(index, data)| async move { + if index > 0 { + tokio::time::sleep(token_latency).await; + } + Ok::(Event::default().data(data)) + }); + Sse::new(events).into_response() } async fn chat(State(state): State, Json(body): Json) -> Response { @@ -154,7 +205,12 @@ async fn chat(State(state): State, Json(body): Json) -> Res } if body.get("stream") == Some(&Value::Bool(true)) { - return stream(model, marker == Some("truncated_stream")); + return stream( + model, + marker == Some("truncated_stream"), + requested_output_tokens(&body), + state.token_latency, + ); } Json(completion(model, "OK")).into_response() } @@ -169,6 +225,7 @@ async fn run(args: Args) -> Result<(), String> { let address = listener.local_addr().map_err(|error| error.to_string())?; let state = BackendState { latency: Duration::from_millis(args.latency_ms), + token_latency: Duration::from_millis(args.token_latency_ms), attempts: Arc::new(Mutex::new(HashMap::new())), }; let app = Router::new() @@ -194,6 +251,23 @@ async fn run(args: Args) -> Result<(), String> { .map_err(|error| error.to_string()) } +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::requested_output_tokens; + + #[test] + fn output_tokens_follow_openai_limits_and_stay_bounded() { + assert_eq!(requested_output_tokens(&json!({"max_tokens": 512})), 512); + assert_eq!( + requested_output_tokens(&json!({"max_completion_tokens": 8_192})), + 4_096 + ); + assert_eq!(requested_output_tokens(&json!({})), 2); + } +} + #[tokio::main] async fn main() -> ExitCode { match run(Args::parse()).await { diff --git a/docs/operations/soak_test.md b/docs/operations/soak_test.md index 3c41d9753..e66627109 100644 --- a/docs/operations/soak_test.md +++ b/docs/operations/soak_test.md @@ -82,6 +82,8 @@ The command runs oha and AIPerf sequentially for every model id, then writes `re ```bash python3.12 scripts/benchmark_routing_algorithms.py \ --base-url http://127.0.0.1:4000 \ + --direct-base-url http://127.0.0.1:8100 \ + --direct-model mock/weak \ --model noop=switchyard/noop \ --model passthrough=switchyard/passthrough \ --model random=switchyard/random \ @@ -95,14 +97,30 @@ python3.12 scripts/benchmark_routing_algorithms.py \ --backend-label "release model deployment" ``` +`--direct-base-url` and `--direct-model` add an AIPerf arm that calls the backend without +Switchyard. The report writes the end-to-end latency and throughput difference for every routed +arm. That difference isolates Switchyard overhead only when the direct and routed arms use the +same backend deployment, model, and response settings. Resilience scenarios have different +failure semantics, so the script does not run or compare the direct arm for them. For an +authenticated backend, pass the name of the key variable, not the key itself, with +`--direct-api-key-env NVIDIA_API_KEY`. + +The Markdown report starts with one overhead row for each route and workload. Positive request or +TTFT latency means the routed request took longer than the direct request. Negative request or +token throughput means the routed path processed less work. The `standard` scenario set includes +all non-resilience request patterns exported by the Rust crate: short and long contexts, long +outputs, shared prefixes, mixed request sizes, growing conversations, large tool catalogs, +tool-call bursts, stage changes, and easy/hard classifier mixes. + The Rust crate exports one AIPerf `inputs-json` file per scenario. oha reuses the first exported `short-interactive` payload as a non-streaming fixed body and reports the raw HTTP request rate and latency ceiling. AIPerf replays every selected streaming session and reports request latency, time to first token (TTFT), inter-token latency (ITL), request throughput, output-token throughput, and -multi-run confidence intervals. The command saves `/v1/stats` before and after each AIPerf run so -the report also records selected-target calls and shares, classifier calls/errors and latency, and -mean routing overhead. It runs -jobs sequentially because simultaneous tools would compete for the same server capacity. +multi-run confidence intervals. The command saves `/v1/stats` before and after each scenario/load +cell, so the routing calls and errors total all independent repetitions in that cell. The report +also records selected-target shares, classifier calls/errors and latency, and mean routing +overhead. It runs jobs sequentially because simultaneous tools would compete for the same server +capacity. Use the load schedules after the fixed scenario comparison establishes a baseline: @@ -150,9 +168,15 @@ Install [oha](https://github.com/hatoo/oha) and ```bash cargo install oha -uv tool install --python 3.12 aiperf +uv tool install --python 3.12 'aiperf==0.11.0' ``` +The benchmark checks the AIPerf version before it creates an output directory. It runs each +repetition in a separate AIPerf process and calculates confidence intervals from those independent +runs. Each process starts one required record processor before profiling. The benchmark gives each +process a deadline based on its request schedule, then stops the whole process group if that deadline +expires. This prevents a startup failure from leaving a benchmark stuck at zero processed records. + Then run the local test from the repository root: ```bash @@ -162,6 +186,11 @@ python3.12 scripts/run_local_soak_test.py \ --request-count 100 ``` +The local backend waits 40 ms before the first token and 1 ms between output tokens by default. +It streams the output limit from each Rust scenario, so decode-heavy requests produce 512 or 1,024 +tokens and AIPerf can measure TTFT, ITL, and token throughput separately. Use +`--mock-latency-ms` and `--mock-token-latency-ms` to change that deterministic timing model. + `--duration` controls how long the Rust soak tester runs. `--request-count` controls how many measured requests each load tool sends to each algorithm. `--help` explains every flag. Set `OHA_BIN`, `AIPERF_BIN`, diff --git a/scripts/aiperf_runner.py b/scripts/aiperf_runner.py new file mode 100644 index 000000000..f83165df1 --- /dev/null +++ b/scripts/aiperf_runner.py @@ -0,0 +1,220 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Run the supported AIPerf release without allowing its startup race to hang.""" + +import json +import math +import os +import signal +import statistics +import subprocess +import time +from collections.abc import Sequence +from pathlib import Path + +SUPPORTED_AIPERF_VERSION = "0.11.0" +PROCESS_GROUP_GRACE_SECONDS = 5 +_T_CRITICAL_95 = { + 2: 12.706, + 3: 4.303, + 4: 3.182, + 5: 2.776, + 6: 2.571, + 7: 2.447, + 8: 2.365, + 9: 2.306, + 10: 2.262, +} +_REQUIRED_STATISTICS = ( + ("error_request_count", "avg"), + ("request_count", "avg"), + ("request_throughput", "avg"), +) + + +def validate_aiperf_version(binary: str) -> None: + """Reject AIPerf versions whose CLI or startup behavior is not covered.""" + try: + result = subprocess.run( + [binary, "--version"], + capture_output=True, + text=True, + check=False, + timeout=10, + ) + except (OSError, subprocess.TimeoutExpired) as error: + raise RuntimeError(f"could not read the AIPerf version from {binary}: {error}") from error + version = result.stdout.strip() + if result.returncode != 0 or version != SUPPORTED_AIPERF_VERSION: + found = version or result.stderr.strip() or "unknown" + raise RuntimeError( + f"AIPerf {found} is unsupported; install {SUPPORTED_AIPERF_VERSION} with: " + f"uv tool install --python 3.12 'aiperf=={SUPPORTED_AIPERF_VERSION}'" + ) + + +def _process_group_exists(process_group: int) -> bool: + try: + os.killpg(process_group, 0) + except ProcessLookupError: + return False + except PermissionError: + # A completed leader's process-group id can be reused by a process we do not own. + return False + return True + + +def _stop_process_group(process: subprocess.Popen[bytes]) -> None: + """Stop AIPerf and every worker process it started.""" + process_group = process.pid + try: + os.killpg(process_group, signal.SIGTERM) + except ProcessLookupError: + pass + except PermissionError as error: + if process.poll() is None: + raise RuntimeError( + f"could not stop running AIPerf process group {process_group}" + ) from error + return + deadline = time.monotonic() + PROCESS_GROUP_GRACE_SECONDS + while _process_group_exists(process_group) and time.monotonic() < deadline: + process.poll() + time.sleep(0.05) + if _process_group_exists(process_group): + try: + os.killpg(process_group, signal.SIGKILL) + except ProcessLookupError: + pass + except PermissionError as error: + if process.poll() is None: + raise RuntimeError( + f"could not kill running AIPerf process group {process_group}" + ) from error + try: + process.wait(timeout=PROCESS_GROUP_GRACE_SECONDS) + except subprocess.TimeoutExpired as error: + raise RuntimeError(f"could not reap AIPerf process {process.pid}") from error + + +def run_profile( + command: Sequence[str], + log_path: Path, + artifact_dir: Path, + timeout_seconds: int, +) -> Path: + """Run one bounded AIPerf process and return its verified export.""" + artifact_dir.parent.mkdir(parents=True, exist_ok=True) + with log_path.open("wb") as log: + try: + process = subprocess.Popen( + [*command, "--artifact-dir", str(artifact_dir)], + stdout=log, + stderr=subprocess.STDOUT, + start_new_session=True, + ) + except OSError as error: + raise RuntimeError(f"could not start AIPerf: {error}") from error + stopped = False + try: + try: + returncode = process.wait(timeout=timeout_seconds) + except subprocess.TimeoutExpired as error: + timeout_message = f"AIPerf exceeded its {timeout_seconds}-second run limit" + log.write(f"\n{timeout_message}.\n".encode()) + log.flush() + _stop_process_group(process) + stopped = True + raise RuntimeError(f"{timeout_message}; see {log_path}") from error + finally: + if not stopped and (process.poll() is None or _process_group_exists(process.pid)): + _stop_process_group(process) + if returncode != 0: + raise RuntimeError(f"AIPerf failed with status {returncode}; see {log_path}") + export_path = artifact_dir / "profile_export_aiperf.json" + if not export_path.is_file(): + raise RuntimeError(f"AIPerf did not write {export_path.name}; see {log_path}") + return export_path + + +def _summary(values: Sequence[float], unit: str) -> dict[str, float | str | None]: + """Return the 95% confidence summary used by the benchmark report.""" + count = len(values) + mean = statistics.fmean(values) + standard_deviation = statistics.stdev(values) if count > 1 else 0.0 + standard_error = standard_deviation / math.sqrt(count) + critical = _T_CRITICAL_95.get(count) + margin = critical * standard_error if critical is not None else 0.0 + return { + "mean": mean, + "std": standard_deviation, + "min": min(values), + "max": max(values), + "cv": standard_deviation / mean if mean else 0.0, + "se": standard_error, + "ci_low": mean - margin, + "ci_high": mean + margin, + "t_critical": critical, + "unit": unit, + } + + +def aggregate_exports(exports: Sequence[Path], output_path: Path) -> Path: + """Combine independent AIPerf exports into the aggregate schema consumed by the report.""" + if not 2 <= len(exports) <= 10: + raise RuntimeError("AIPerf aggregation requires between 2 and 10 exports") + collected: dict[tuple[str, str, str], list[float]] = {} + metric_units: dict[tuple[str, str], str] = {} + for path in exports: + try: + document = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + raise RuntimeError(f"could not read AIPerf export {path}: {error}") from error + if not isinstance(document, dict): + raise RuntimeError(f"AIPerf export is not a JSON object: {path}") + version = document.get("aiperf_version") + if version != SUPPORTED_AIPERF_VERSION: + raise RuntimeError( + f"AIPerf export {path} has version {version!r}; expected {SUPPORTED_AIPERF_VERSION}" + ) + error_count = document.get("error_request_count") + clean_error_summary = document.get("error_summary") == [] + if error_count is None and ("error_request_count" in document or clean_error_summary): + document["error_request_count"] = {"unit": "requests", "avg": 0.0} + for metric_name, statistic in _REQUIRED_STATISTICS: + metric = document.get(metric_name) + value = metric.get(statistic) if isinstance(metric, dict) else None + unit = metric.get("unit") if isinstance(metric, dict) else None + if ( + not isinstance(value, int | float) + or isinstance(value, bool) + or not isinstance(unit, str) + ): + raise RuntimeError(f"AIPerf export {path} has no numeric {metric_name}.{statistic}") + for metric_name, metric in document.items(): + if not isinstance(metric, dict) or not isinstance(metric.get("unit"), str): + continue + unit = metric["unit"] + for statistic, value in metric.items(): + if statistic == "unit" or isinstance(value, bool): + continue + if isinstance(value, int | float): + previous_unit = metric_units.setdefault((metric_name, statistic), unit) + if unit != previous_unit: + raise RuntimeError( + "AIPerf exports disagree on the unit for " + f"{metric_name}.{statistic}: {previous_unit!r} and {unit!r}" + ) + collected.setdefault((metric_name, statistic, unit), []).append(float(value)) + metrics = { + f"{metric}_{statistic}": _summary(values, unit) + for (metric, statistic, unit), values in collected.items() + if len(values) == len(exports) + } + output_path.parent.mkdir(parents=True, exist_ok=False) + output_path.write_text( + f"{json.dumps({'aiperf_version': SUPPORTED_AIPERF_VERSION, 'metrics': metrics}, indent=2)}\n", + encoding="utf-8", + ) + return output_path diff --git a/scripts/benchmark_routing_algorithms.py b/scripts/benchmark_routing_algorithms.py index aaac6dab0..635ec181a 100755 --- a/scripts/benchmark_routing_algorithms.py +++ b/scripts/benchmark_routing_algorithms.py @@ -14,10 +14,19 @@ import urllib.error import urllib.request from collections.abc import Sequence -from dataclasses import asdict, dataclass +from dataclasses import asdict, dataclass, replace from datetime import datetime, timezone from pathlib import Path +if __package__: + from .aiperf_runner import aggregate_exports, run_profile, validate_aiperf_version +else: + from aiperf_runner import aggregate_exports, run_profile, validate_aiperf_version + +MAX_MATERIALIZED_REQUESTS = 1_000_000 +MAX_MATERIALIZED_BYTES = 256 * 1024 * 1024 +AIPERF_LIFECYCLE_ALLOWANCE_SECONDS = 120 + @dataclass(frozen=True) class RequiredBinary: @@ -29,6 +38,15 @@ class RequiredBinary: setup: str +@dataclass(frozen=True) +class DirectBaseline: + """A backend endpoint that AIPerf calls without going through Switchyard.""" + + base_url: str + model: str + api_key_env: str | None = None + + @dataclass(frozen=True) class BenchmarkConfig: """Inputs shared by every algorithm comparison run.""" @@ -52,6 +70,18 @@ class BenchmarkConfig: context_window_tokens: int = 32_768 request_rate: float | None = None scenario_backend_reset_url: str | None = None + direct_baseline: DirectBaseline | None = None + + +@dataclass(frozen=True) +class BenchmarkArm: + """One endpoint and model pair included in the comparison.""" + + label: str + model: str + base_url: str + bypasses_switchyard: bool + api_key_env: str | None = None @dataclass(frozen=True) @@ -88,6 +118,7 @@ class BenchmarkResult: algorithm: str model: str scenario: str + scenario_description: str scenario_group: str load_profile: str expected_behavior: str @@ -116,6 +147,12 @@ class BenchmarkResult: classifier_errors: int classifier_latency_avg_ms: float | None routing_overhead_avg_ms: float | None + bypasses_switchyard: bool + aiperf_request_throughput_delta_pct: float | None = None + aiperf_request_latency_p50_delta_ms: float | None = None + aiperf_ttft_p50_delta_ms: float | None = None + aiperf_ttft_p99_delta_ms: float | None = None + aiperf_output_tokens_per_second_delta_pct: float | None = None def positive_int(value: str) -> int: @@ -152,6 +189,18 @@ def parser() -> argparse.ArgumentParser: formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) command.add_argument("--base-url", required=True, help="URL of a running Switchyard server") + command.add_argument( + "--direct-base-url", + help="backend URL for an AIPerf baseline that bypasses Switchyard", + ) + command.add_argument( + "--direct-model", + help="backend model id for the direct baseline; requires --direct-base-url", + ) + command.add_argument( + "--direct-api-key-env", + help="environment variable containing the direct backend API key", + ) command.add_argument( "--model", action="append", @@ -303,18 +352,19 @@ def reset_scenario_backend(url: str | None) -> None: raise RuntimeError(f"scenario backend reset returned an invalid response from {url}") -def _nested_number(document: dict[str, object], *keys: str) -> float: - current: object = document - for key in keys: - current = current.get(key) if isinstance(current, dict) else None - return float(current) if isinstance(current, int | float) else 0.0 +def _nested_number(document: dict[str, object], section: str, field: str) -> float: + section_value = document.get(section) + value = section_value.get(field) if isinstance(section_value, dict) else None + return float(value) if isinstance(value, int | float) else 0.0 def _histogram_delta( - before: dict[str, object], after: dict[str, object], *keys: str + before: dict[str, object], after: dict[str, object], histogram: str ) -> float | None: - count = _nested_number(after, *keys, "count") - _nested_number(before, *keys, "count") - total = _nested_number(after, *keys, "total_ms") - _nested_number(before, *keys, "total_ms") + count = _nested_number(after, histogram, "count") - _nested_number(before, histogram, "count") + total = _nested_number(after, histogram, "total_ms") - _nested_number( + before, histogram, "total_ms" + ) return total / count if count > 0 else None @@ -578,8 +628,7 @@ def _latency(document: dict[str, object], metric: str, statistic: str, path: Pat def parse_result( - algorithm: str, - model: str, + arm: BenchmarkArm, scenario: ScenarioDefinition, load_profile: str, oha_path: Path | None, @@ -593,7 +642,7 @@ def parse_result( oha = read_json_object(oha_path) success, _unit = _metric(oha, "summary", "successRate") if success != 1: - raise RuntimeError(f"oha success rate for {model} was {success}; see {oha_path}") + raise RuntimeError(f"oha success rate for {arm.model} was {success}; see {oha_path}") oha_rps, _unit = _metric(oha, "summary", "requestsPerSec") p50, _unit = _metric(oha, "latencyPercentiles", "p50") p99, _unit = _metric(oha, "latencyPercentiles", "p99") @@ -620,9 +669,10 @@ def parse_result( ) routing = stats_delta(before_stats, after_stats) return BenchmarkResult( - algorithm=algorithm, - model=model, + algorithm=arm.label, + model=arm.model, scenario=scenario.id, + scenario_description=scenario.description, scenario_group=scenario.group, load_profile=load_profile, expected_behavior=scenario.expected, @@ -666,14 +716,100 @@ def parse_result( classifier_errors=routing.classifier_errors, classifier_latency_avg_ms=routing.classifier_latency_avg_ms, routing_overhead_avg_ms=routing.routing_overhead_avg_ms, + bypasses_switchyard=arm.bypasses_switchyard, ) +def _percent_change(value: float | None, baseline: float | None) -> float | None: + if value is None or baseline is None or baseline == 0: + return None + return (value / baseline - 1) * 100 + + +def _difference(value: float | None, baseline: float | None) -> float | None: + if value is None or baseline is None: + return None + return value - baseline + + +def _direct_baselines( + results: Sequence[BenchmarkResult], +) -> dict[tuple[str, str], BenchmarkResult]: + """Index the one direct-backend result allowed for each workload.""" + baselines = {} + for result in results: + if not result.bypasses_switchyard: + continue + key = (result.scenario, result.load_profile) + if key in baselines: + raise RuntimeError( + f"duplicate direct baseline for {result.scenario} {result.load_profile}" + ) + baselines[key] = result + return baselines + + +def compare_to_direct_backend(results: Sequence[BenchmarkResult]) -> list[BenchmarkResult]: + """Attach AIPerf deltas from the direct-backend row for each workload.""" + baselines = _direct_baselines(results) + if not baselines: + return list(results) + + compared = [] + for result in results: + if result.bypasses_switchyard: + compared.append(result) + continue + key = (result.scenario, result.load_profile) + baseline = baselines.get(key) + if baseline is None: + if result.scenario_group != "resilience": + raise RuntimeError( + f"missing direct baseline for {result.scenario} {result.load_profile}" + ) + compared.append(result) + continue + if result.aiperf_error_rate != 0 or baseline.aiperf_error_rate != 0: + compared.append(result) + continue + compared.append( + replace( + result, + aiperf_request_throughput_delta_pct=_percent_change( + result.aiperf_requests_per_second, + baseline.aiperf_requests_per_second, + ), + aiperf_request_latency_p50_delta_ms=_difference( + result.aiperf_request_latency_p50_ms, + baseline.aiperf_request_latency_p50_ms, + ), + aiperf_ttft_p50_delta_ms=_difference( + result.aiperf_ttft_p50_ms, + baseline.aiperf_ttft_p50_ms, + ), + aiperf_ttft_p99_delta_ms=_difference( + result.aiperf_ttft_p99_ms, + baseline.aiperf_ttft_p99_ms, + ), + aiperf_output_tokens_per_second_delta_pct=_percent_change( + result.aiperf_output_tokens_per_second, + baseline.aiperf_output_tokens_per_second, + ), + ) + ) + return compared + + def format_metric(value: float | None) -> str: """Render a report value without implying precision the benchmark lacks.""" return "n/a" if value is None else f"{value:.2f}" +def format_delta(value: float | None, suffix: str = "") -> str: + """Render a signed direct-backend comparison.""" + return "n/a" if value is None else f"{value:+.2f}{suffix}" + + def write_report(config: BenchmarkConfig, results: Sequence[BenchmarkResult]) -> None: """Write machine-readable and reviewer-readable comparison reports.""" if not results: @@ -683,6 +819,9 @@ def write_report(config: BenchmarkConfig, results: Sequence[BenchmarkResult]) -> "generated_at": datetime.now(timezone.utc).isoformat(), "backend": config.backend_label, "base_url": config.base_url, + "direct_baseline": ( + asdict(config.direct_baseline) if config.direct_baseline is not None else None + ), "concurrency": config.concurrency, "request_count": config.request_count, "profile_runs": config.profile_runs, @@ -700,21 +839,90 @@ def write_report(config: BenchmarkConfig, results: Sequence[BenchmarkResult]) -> writer.writeheader() writer.writerows(rows) - lines = [ - "# Routing algorithm performance", - "", + lines = ["# Routing algorithm performance", ""] + run_details = [ f"- Backend: {config.backend_label}", f"- AIPerf repetitions per row: {config.profile_runs}", f"- Scenario set: {config.scenario_set}", "- Run order: algorithms ran back-to-back within each scenario and load; all jobs " "remained sequential", - "", - "## Throughput and latency", - "", - "| Algorithm | Scenario | Load | oha req/s | AIPerf req/s | request p50 ms | " - "TTFT p50 ms | TTFT p99 ms | ITL p50 ms | output tok/s | error rate | gate |", - "|---|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---|", ] + if config.direct_baseline is not None: + run_details.append( + f"- Direct baseline: `{config.direct_baseline.model}` at " + f"`{config.direct_baseline.base_url}`; this arm bypasses Switchyard" + ) + overhead_results = [ + result + for result in results + if not result.bypasses_switchyard and result.scenario_group != "resilience" + ] + lines.extend( + ( + "## Routing overhead versus the direct backend", + "", + "The direct arm sends the same Rust-exported requests without Switchyard. These " + "deltas isolate Switchyard overhead only when both arms use the same backend " + "deployment, model, and response settings. Positive latency means the routed " + "request took longer. Negative throughput means the routed path processed less " + "work. Resilience workloads are shown later but are not compared because their " + "failure behavior differs.", + "", + ) + ) + if not overhead_results: + lines.append("No successful direct-backend comparisons were available.") + else: + lines.extend( + ( + "| Workload | Route | Load | Request p50 | TTFT p50 | Request throughput | " + "Token throughput |", + "|---|---|---|---:|---:|---:|---:|", + ) + ) + for result in overhead_results: + lines.append( + f"| {result.scenario.replace('-', ' ')} | {result.algorithm} | " + f"{result.load_profile} | " + f"{format_delta(result.aiperf_request_latency_p50_delta_ms, ' ms')} | " + f"{format_delta(result.aiperf_ttft_p50_delta_ms, ' ms')} | " + f"{format_delta(result.aiperf_request_throughput_delta_pct, '%')} | " + f"{format_delta(result.aiperf_output_tokens_per_second_delta_pct, '%')} |" + ) + lines.extend( + ( + "", + "Use the confidence intervals in the detailed results before treating a small " + "throughput change as meaningful. A small negative latency or positive " + "throughput delta is run-to-run noise when those intervals overlap. Request and " + "token throughput percentage changes match when every arm emits the same fixed " + "output length; the detailed table shows the absolute token rates.", + "", + "## Workloads measured", + "", + "| Workload | Request pattern |", + "|---|---|", + ) + ) + seen_scenarios = set() + for result in overhead_results: + if result.scenario in seen_scenarios: + continue + seen_scenarios.add(result.scenario) + lines.append( + f"| {result.scenario.replace('-', ' ')} | {result.scenario_description} |" + ) + lines.extend(("", "## Run details", "", *run_details)) + lines.extend( + ( + "", + "## Throughput and latency", + "", + "| Algorithm | Scenario | Load | oha req/s | AIPerf req/s | request p50 ms | " + "TTFT p50 ms | TTFT p99 ms | ITL p50 ms | output tok/s | error rate | gate |", + "|---|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---|", + ) + ) for result in results: if result.scenario_group == "resilience": continue @@ -730,17 +938,54 @@ def write_report(config: BenchmarkConfig, results: Sequence[BenchmarkResult]) -> f"{result.aiperf_error_rate:.2%} | " f"{'PASS' if result.expectation_met else 'FAIL'} |" ) + if config.profile_runs > 1: + lines.extend( + ( + "", + "## Repeatability", + "", + "The script calculates variation and confidence intervals across independent " + "AIPerf runs. " + "Treat a small throughput delta as noise when these intervals overlap.", + "", + "| Algorithm | Scenario | Load | req/s average | req/s CV | req/s 95% CI |", + "|---|---|---|---:|---:|---:|", + ) + ) + for result in results: + if result.scenario_group == "resilience": + continue + confidence_interval = ( + "n/a" + if result.aiperf_request_throughput_ci_low is None + or result.aiperf_request_throughput_ci_high is None + else f"{result.aiperf_request_throughput_ci_low:.2f}–" + f"{result.aiperf_request_throughput_ci_high:.2f}" + ) + coefficient_of_variation = ( + "n/a" + if result.aiperf_request_throughput_cv is None + else f"{result.aiperf_request_throughput_cv:.2%}" + ) + lines.append( + f"| {result.algorithm} | {result.scenario} | {result.load_profile} | " + f"{format_metric(result.aiperf_requests_per_second)} | " + f"{coefficient_of_variation} | {confidence_interval} |" + ) lines.extend( ( "", "## Routing behavior", "", "| Algorithm | Scenario | Load | selected target calls | selected target share | target errors | " - "classifier calls | classifier errors | classifier avg ms | routing avg ms |", + "classifier calls | classifier errors | classifier avg ms | " + "Switchyard routing time avg ms |", "|---|---|---|---|---|---|---:|---:|---:|---:|", ) ) for result in results: + if result.bypasses_switchyard: + continue lines.append( f"| {result.algorithm} | {result.scenario} | {result.load_profile} | " f"`{result.selected_model_calls}` | `{result.selected_model_share}` | " @@ -763,6 +1008,8 @@ def write_report(config: BenchmarkConfig, results: Sequence[BenchmarkResult]) -> ) ) for result in resilience: + if result.bypasses_switchyard: + continue lines.append( f"| {result.algorithm} | {result.scenario} | {result.load_profile} | " f"{result.aiperf_error_rate:.2%} | " @@ -834,50 +1081,123 @@ def run_oha( return result_path +def materialize_aiperf_input( + scenario: ScenarioDefinition, + output_path: Path, + minimum_request_count: int, + namespace: str = "load", +) -> Path: + """Repeat Rust-owned sessions with unique ids so AIPerf never wraps the dataset.""" + if minimum_request_count > MAX_MATERIALIZED_REQUESTS: + raise RuntimeError( + f"AIPerf input needs {minimum_request_count} requests; maximum is " + f"{MAX_MATERIALIZED_REQUESTS}" + ) + document = read_json_object(scenario.input_file) + data = document.get("data") + if not isinstance(data, list) or not data: + raise RuntimeError(f"AIPerf input has no sessions: {scenario.input_file}") + + validated = [] + requests_per_replica = 0 + bytes_per_replica = 0 + for raw_session in data: + if not isinstance(raw_session, dict): + raise RuntimeError(f"invalid AIPerf session in {scenario.input_file}") + session_id = raw_session.get("session_id") + payloads = raw_session.get("payloads") + if not isinstance(session_id, str) or not isinstance(payloads, list) or not payloads: + raise RuntimeError(f"invalid AIPerf session in {scenario.input_file}") + validated.append((raw_session, session_id, len(payloads))) + requests_per_replica += len(payloads) + bytes_per_replica += len(json.dumps(raw_session).encode()) + 64 + replica_count = math.ceil(minimum_request_count / requests_per_replica) + estimated_bytes = replica_count * bytes_per_replica + if estimated_bytes > MAX_MATERIALIZED_BYTES: + raise RuntimeError( + f"AIPerf input would use about {estimated_bytes} bytes; maximum is " + f"{MAX_MATERIALIZED_BYTES}" + ) + + sessions = [] + request_count = 0 + replica = 0 + for _replica_index in range(replica_count): + for raw_session, session_id, payload_count in validated: + sessions.append( + { + **raw_session, + "session_id": f"{session_id}-{namespace}-{replica:06d}", + } + ) + request_count += payload_count + replica += 1 + if request_count >= minimum_request_count: + break + if request_count >= minimum_request_count: + break + output_path.write_text(json.dumps({"data": sessions}), encoding="utf-8") + return output_path + + +def profile_request_count(config: BenchmarkConfig, profile: dict[str, object]) -> int: + """Return a conservative request bound for one AIPerf schedule.""" + if profile.get("kind") != "traffic_burst": + return config.request_count + duration = profile.get("duration_seconds") + points = profile.get("points") + if not isinstance(duration, int) or not isinstance(points, list): + raise RuntimeError("traffic-burst profile is missing its duration or points") + parsed_points = [] + for point in points: + if not isinstance(point, dict): + raise RuntimeError("traffic-burst profile has an invalid point") + time_s = point.get("time_s") + multiplier = point.get("rate_multiplier") + if not isinstance(time_s, int | float) or not isinstance(multiplier, int | float): + raise RuntimeError("traffic-burst profile has an invalid point") + parsed_points.append((float(time_s), float(multiplier))) + request_multiplier_seconds = sum( + (right_time - left_time) * (left_rate + right_rate) / 2 + for (left_time, left_rate), (right_time, right_rate) in zip( + parsed_points, parsed_points[1:], strict=False + ) + ) + return math.ceil(request_multiplier_seconds * (config.request_rate or config.concurrency)) + + +def aiperf_timeout_seconds( + config: BenchmarkConfig, + scenario: ScenarioDefinition, + profile: dict[str, object], + concurrency: int | None, +) -> int: + """Bound one process by its schedule and per-request timeout.""" + request_timeout = 1 if scenario.id == "client-cancellation" else 30 + if profile.get("kind") == "traffic_burst": + duration = profile.get("duration_seconds") + if not isinstance(duration, int): + raise RuntimeError(f"traffic-burst profile has no duration for {scenario.id}") + workload_seconds = duration + request_timeout + else: + effective_concurrency = concurrency or config.concurrency + workload_seconds = math.ceil(config.request_count / effective_concurrency) * request_timeout + return AIPERF_LIFECYCLE_ALLOWANCE_SECONDS + workload_seconds + + def run_aiperf( config: BenchmarkConfig, - label: str, - model: str, + arm: BenchmarkArm, scenario: ScenarioDefinition, profile: dict[str, object], output_dir: Path, + artifact_label: str, concurrency: int | None = None, ) -> Path: - """Run one isolated AIPerf profile using a Rust-exported inputs-json dataset.""" - artifact_dir = output_dir / label - command = [ - config.aiperf_bin, - "profile", - "--model", - model, - "--url", - config.base_url, - "--endpoint-type", - "chat", - "--streaming", - "--tokenizer", - config.tokenizer, - "--custom-dataset-type", - "inputs-json", - "--input-file", - str(scenario.input_file), - "--session-header", - "x-switchyard-session-id", - "--random-seed", - "42", - "--warmup-duration", - "1", - "--num-profile-runs", - str(config.profile_runs), - "--failed-request-threshold", - "1", - "--request-timeout-seconds", - "1" if scenario.id == "client-cancellation" else "30", - "--ui", - "none", - "--artifact-dir", - str(artifact_dir), - ] + """Run independent AIPerf repetitions using Rust-exported inputs-json datasets.""" + artifact_root = output_dir / artifact_label + effective_concurrency = concurrency or config.concurrency + load_arguments: list[str] = [] kind = profile.get("kind") if kind == "traffic_burst": points = profile.get("points") @@ -894,20 +1214,80 @@ def run_aiperf( "qps": base_rate * float(point["rate_multiplier"]), } ) - series_path = output_dir / f"{label}-request-rate.json" + series_path = output_dir / f"{artifact_label}-request-rate.json" series_path.write_text( f"{json.dumps({'points': rate_points}, indent=2)}\n", encoding="utf-8" ) - command.extend(("--request-rate-series", str(series_path), "--arrival-pattern", "constant")) - command.extend(("--benchmark-duration", str(profile.get("duration_seconds")))) - command.extend(("--concurrency", str(config.concurrency))) + load_arguments.extend( + ("--request-rate-series", str(series_path), "--arrival-pattern", "constant") + ) + load_arguments.extend(("--benchmark-duration", str(profile.get("duration_seconds")))) + load_arguments.extend(("--concurrency", str(config.concurrency))) else: - command.extend(("--concurrency", str(concurrency or config.concurrency))) - command.extend(("--request-count", str(config.request_count))) - run_checked(f"AIPerf {label}", command, output_dir / f"{label}.log") - if config.profile_runs > 1: - return artifact_dir / "aggregate" / "profile_export_aiperf_aggregate.json" - return artifact_dir / "profile_export_aiperf.json" + load_arguments.extend(("--concurrency", str(effective_concurrency))) + load_arguments.extend(("--request-count", str(config.request_count))) + + exports: list[Path] = [] + for trial in range(1, config.profile_runs + 1): + run_label = f"{artifact_label}-run-{trial:02d}" + input_path = materialize_aiperf_input( + scenario, + output_dir / f"{run_label}-inputs.json", + profile_request_count(config, profile), + namespace=f"{artifact_label}-run-{trial:02d}", + ) + command = [ + config.aiperf_bin, + "profile", + "--model", + arm.model, + "--url", + arm.base_url, + "--endpoint-type", + "chat", + "--streaming", + "--tokenizer", + config.tokenizer, + "--custom-dataset-type", + "inputs-json", + "--input-file", + str(input_path), + "--session-header", + "x-switchyard-session-id", + "--random-seed", + "42", + "--num-profile-runs", + "1", + "--record-processors", + "1", + "--request-timeout-seconds", + "1" if scenario.id == "client-cancellation" else "30", + "--ui", + "none", + *load_arguments, + ] + if arm.api_key_env is not None: + command.extend(("--api-key", f"${{{arm.api_key_env}}}")) + trial_root = ( + artifact_root / "profile_runs" / f"run_{trial:04d}" + if config.profile_runs > 1 + else artifact_root + ) + exports.append( + run_profile( + command, + output_dir / f"{run_label}.log", + trial_root, + aiperf_timeout_seconds(config, scenario, profile, concurrency), + ) + ) + if len(exports) == 1: + return exports[0] + aggregate_path: Path = aggregate_exports( + exports, + artifact_root / "aggregate" / "profile_export_aiperf_aggregate.json", + ) + return aggregate_path def run_benchmark(config: BenchmarkConfig) -> Path: @@ -916,7 +1296,20 @@ def run_benchmark(config: BenchmarkConfig) -> Path: raise RuntimeError("at least one algorithm model is required") if config.profile_runs > 10: raise RuntimeError("--profile-runs must be between 1 and 10") - labels = [label for label, _model in config.models] + validate_aiperf_version(config.aiperf_bin) + arms = [BenchmarkArm(label, model, config.base_url, False) for label, model in config.models] + if config.direct_baseline is not None: + arms.insert( + 0, + BenchmarkArm( + "direct-backend", + config.direct_baseline.model, + config.direct_baseline.base_url, + True, + config.direct_baseline.api_key_env, + ), + ) + labels = [arm.label for arm in arms] if len(labels) != len(set(labels)): raise RuntimeError("algorithm labels must be unique") config.output_dir.mkdir(parents=True, exist_ok=False) @@ -925,10 +1318,7 @@ def run_benchmark(config: BenchmarkConfig) -> Path: oha_dir.mkdir() aiperf_dir.mkdir() results = [] - definition_sets = [ - export_scenarios(config, index, model) - for index, (_algorithm, model) in enumerate(config.models) - ] + definition_sets = [export_scenarios(config, index, arm.model) for index, arm in enumerate(arms)] expected_ids = [scenario.id for scenario in definition_sets[0]] for definitions in definition_sets[1:]: if [scenario.id for scenario in definitions] != expected_ids: @@ -950,36 +1340,54 @@ def run_benchmark(config: BenchmarkConfig) -> Path: ) for concurrency in concurrencies: load_label = profile_id if concurrency is None else f"{profile_id}@{concurrency}" - for index, (algorithm, model) in enumerate(config.models): + for index, arm in enumerate(arms): scenario = definition_sets[index][scenario_index] + if arm.bypasses_switchyard and scenario.group == "resilience": + continue artifact_label = ( f"algorithm-{index:02d}-{scenario.id}-{load_label.replace('@', '-')}" ) oha_path = None - if scenario.id == "short-interactive" and profile_id == "fixed": - oha_path = run_oha(config, artifact_label, scenario, oha_dir) + if ( + not arm.bypasses_switchyard + and scenario.id == "short-interactive" + and profile_id == "fixed" + ): + oha_path = run_oha( + config, + artifact_label, + scenario, + oha_dir, + ) reset_scenario_backend(config.scenario_backend_reset_url) - before = capture_stats( - config.base_url, - aiperf_dir / f"{artifact_label}-stats-before.json", + before = ( + {} + if arm.bypasses_switchyard + else capture_stats( + config.base_url, + aiperf_dir / f"{artifact_label}-stats-before.json", + ) ) aiperf_path = run_aiperf( config, - artifact_label, - model, + arm, scenario, profile, aiperf_dir, + artifact_label, concurrency, ) - after = capture_stats( - config.base_url, - aiperf_dir / f"{artifact_label}-stats-after.json", + after = ( + {} + if arm.bypasses_switchyard + else capture_stats( + config.base_url, + aiperf_dir / f"{artifact_label}-stats-after.json", + ) ) results.append( parse_result( - algorithm, - model, + arm, scenario, load_label, oha_path, @@ -988,6 +1396,7 @@ def run_benchmark(config: BenchmarkConfig) -> Path: after, ) ) + results = compare_to_direct_backend(results) write_report(config, results) failures = [result for result in results if not result.expectation_met] if failures: @@ -1007,7 +1416,12 @@ def default_output_dir() -> Path: def main(argv: Sequence[str] | None = None) -> int: """Parse arguments, run the comparison, and print one actionable failure.""" - args = parser().parse_args(argv) + command = parser() + args = command.parse_args(argv) + if (args.direct_base_url is None) != (args.direct_model is None): + command.error("--direct-base-url and --direct-model must be used together") + if args.direct_api_key_env is not None and args.direct_base_url is None: + command.error("--direct-api-key-env requires --direct-base-url and --direct-model") try: binaries = resolve_binaries( ( @@ -1018,7 +1432,7 @@ def main(argv: Sequence[str] | None = None) -> int: "AIPerf", "AIPERF_BIN", args.aiperf_bin, - "Install AIPerf with: uv tool install --python 3.12 aiperf", + "Install AIPerf with: uv tool install --python 3.12 'aiperf==0.11.0'", ), RequiredBinary( "switchyard-soak", @@ -1049,6 +1463,15 @@ def main(argv: Sequence[str] | None = None) -> int: aiperf_bin=binaries["AIPerf"], soak_bin=binaries["switchyard-soak"], scenario_backend_reset_url=args.scenario_backend_reset_url, + direct_baseline=( + DirectBaseline( + args.direct_base_url, + args.direct_model, + args.direct_api_key_env, + ) + if args.direct_base_url is not None and args.direct_model is not None + else None + ), ) ) except (OSError, RuntimeError) as error: diff --git a/scripts/run_local_soak_test.py b/scripts/run_local_soak_test.py index 2d768a45d..32decdae3 100755 --- a/scripts/run_local_soak_test.py +++ b/scripts/run_local_soak_test.py @@ -20,6 +20,7 @@ from benchmark_routing_algorithms import ( BenchmarkConfig, + DirectBaseline, RequiredBinary, positive_int, resolve_binaries, @@ -93,7 +94,13 @@ def parser() -> argparse.ArgumentParser: "--mock-latency-ms", type=nonnegative_int, default=40, - help="response latency the local scenario backend adds to ordinary calls", + help="latency before the local scenario backend sends its first token", + ) + command.add_argument( + "--mock-token-latency-ms", + type=nonnegative_int, + default=1, + help="latency the local scenario backend adds between streamed output tokens", ) command.add_argument( "--server-port", @@ -243,7 +250,7 @@ def run_local_soak_test(args: argparse.Namespace) -> Path: "AIPerf", "AIPERF_BIN", os.environ.get("AIPERF_BIN", "aiperf"), - "Install AIPerf with: uv tool install --python 3.12 aiperf", + "Install AIPerf with: uv tool install --python 3.12 'aiperf==0.11.0'", ), ) ) @@ -274,6 +281,8 @@ def run_local_soak_test(args: argparse.Namespace) -> Path: str(MOCK_PORT), "--latency-ms", str(args.mock_latency_ms), + "--token-latency-ms", + str(args.mock_token_latency_ms), ], output_dir / "scenario-backend.log", ) @@ -297,7 +306,9 @@ def run_local_soak_test(args: argparse.Namespace) -> Path: concurrency=args.concurrency, request_count=args.request_count, backend_label=( - f"request-aware local backend with {args.mock_latency_ms} ms response latency" + "request-aware local backend with " + f"{args.mock_latency_ms} ms TTFT and " + f"{args.mock_token_latency_ms} ms per output token" ), output_dir=output_dir / "routing-benchmark", oha_bin=oha_bin, @@ -305,6 +316,10 @@ def run_local_soak_test(args: argparse.Namespace) -> Path: soak_bin=soak_bin, profile_runs=1, scenario_backend_reset_url=f"http://127.0.0.1:{MOCK_PORT}/reset", + direct_baseline=DirectBaseline( + base_url=f"http://127.0.0.1:{MOCK_PORT}", + model="mock/weak", + ), ) ) diff --git a/tests/test_aiperf_runner.py b/tests/test_aiperf_runner.py new file mode 100644 index 000000000..71f5fafa7 --- /dev/null +++ b/tests/test_aiperf_runner.py @@ -0,0 +1,157 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import json +import sys +import time + +import pytest + +import scripts.aiperf_runner as aiperf_runner +from scripts.aiperf_runner import aggregate_exports, run_profile, validate_aiperf_version + + +def _write_stubborn_worker(worker) -> None: + worker.write_text( + """import signal +import sys +import time +from pathlib import Path + +signal.signal(signal.SIGTERM, signal.SIG_IGN) +heartbeat = Path(sys.argv[1]) +while True: + heartbeat.write_text(str(time.monotonic())) + time.sleep(0.05) +""" + ) + + +def _assert_heartbeat_stopped(heartbeat) -> None: + heartbeat_before = heartbeat.read_text() + time.sleep(0.2) + assert heartbeat.read_text() == heartbeat_before + + +def test_run_profile_timeout_kills_the_worker_process_group(tmp_path, monkeypatch) -> None: + fake = tmp_path / "fake_aiperf.py" + worker = tmp_path / "worker.py" + child_pid = tmp_path / "child-pid" + heartbeat = tmp_path / "heartbeat" + _write_stubborn_worker(worker) + monkeypatch.setattr(aiperf_runner, "PROCESS_GROUP_GRACE_SECONDS", 0.2) + fake.write_text( + """import subprocess +import sys +import time +from pathlib import Path + +child = subprocess.Popen([sys.executable, sys.argv[2], sys.argv[3]]) +Path(sys.argv[1]).write_text(str(child.pid)) +time.sleep(60) +""" + ) + + started_at = time.monotonic() + with pytest.raises(RuntimeError, match="exceeded its 1-second run limit"): + run_profile( + [sys.executable, str(fake), str(child_pid), str(worker), str(heartbeat)], + tmp_path / "aiperf.log", + tmp_path / "artifacts", + timeout_seconds=1, + ) + + assert time.monotonic() - started_at < 10 + assert child_pid.read_text().isdigit() + _assert_heartbeat_stopped(heartbeat) + assert "exceeded its 1-second run limit" in (tmp_path / "aiperf.log").read_text() + + +def test_run_profile_stops_workers_after_the_leader_exits(tmp_path, monkeypatch) -> None: + fake = tmp_path / "fake_aiperf.py" + worker = tmp_path / "worker.py" + child_pid = tmp_path / "child-pid" + heartbeat = tmp_path / "heartbeat" + _write_stubborn_worker(worker) + monkeypatch.setattr(aiperf_runner, "PROCESS_GROUP_GRACE_SECONDS", 0.2) + fake.write_text( + """import subprocess +import sys +import time +from pathlib import Path + +child = subprocess.Popen([sys.executable, sys.argv[2], sys.argv[3]]) +Path(sys.argv[1]).write_text(str(child.pid)) +while not Path(sys.argv[3]).exists(): + time.sleep(0.01) +""" + ) + + with pytest.raises(RuntimeError, match="did not write profile_export_aiperf.json"): + run_profile( + [sys.executable, str(fake), str(child_pid), str(worker), str(heartbeat)], + tmp_path / "aiperf.log", + tmp_path / "artifacts", + timeout_seconds=10, + ) + + assert child_pid.read_text().isdigit() + _assert_heartbeat_stopped(heartbeat) + + +def test_process_group_probe_ignores_an_unowned_reused_group(monkeypatch) -> None: + def deny_signal(_process_group, _signal) -> None: + raise PermissionError + + monkeypatch.setattr(aiperf_runner.os, "killpg", deny_signal) + + assert not aiperf_runner._process_group_exists(1234) + + +def test_validate_aiperf_version_rejects_uncovered_release(tmp_path) -> None: + fake = tmp_path / "aiperf" + fake.write_text("#!/bin/sh\nprintf '0.12.0\\n'\n") + fake.chmod(0o755) + + with pytest.raises(RuntimeError, match="AIPerf 0.12.0 is unsupported"): + validate_aiperf_version(str(fake)) + + +def test_aggregate_exports_combines_independent_runs(tmp_path) -> None: + exports = [] + for index, throughput in enumerate((10.0, 14.0), start=1): + path = tmp_path / f"run-{index}.json" + document = { + "aiperf_version": "0.11.0", + "error_request_count": None, + "request_count": {"unit": "requests", "avg": 20}, + "request_throughput": { + "unit": "requests/sec", + "avg": throughput, + }, + "time_to_first_token": {"unit": "ms", "p50": 10.0}, + } + if index == 2: + del document["error_request_count"] + document["error_summary"] = [] + path.write_text(json.dumps(document)) + exports.append(path) + + aggregate_path = aggregate_exports(exports, tmp_path / "aggregate" / "report.json") + metrics = json.loads(aggregate_path.read_text())["metrics"] + + assert metrics["request_throughput_avg"]["mean"] == 12.0 + assert metrics["request_throughput_avg"]["ci_low"] < 0 + assert metrics["error_request_count_avg"]["mean"] == 0.0 + + mismatched = json.loads(exports[1].read_text()) + mismatched["time_to_first_token"]["unit"] = "seconds" + exports[1].write_text(json.dumps(mismatched)) + with pytest.raises(RuntimeError, match="disagree on the unit"): + aggregate_exports(exports, tmp_path / "invalid" / "report.json") + + mismatched["time_to_first_token"]["unit"] = "ms" + mismatched["request_count"]["avg"] = True + exports[1].write_text(json.dumps(mismatched)) + with pytest.raises(RuntimeError, match="has no numeric request_count.avg"): + aggregate_exports(exports, tmp_path / "malformed" / "report.json") diff --git a/tests/test_routing_performance_report.py b/tests/test_routing_performance_report.py index e943a38b8..2d9c31f0c 100644 --- a/tests/test_routing_performance_report.py +++ b/tests/test_routing_performance_report.py @@ -3,11 +3,21 @@ import csv import json +from dataclasses import replace +from pathlib import Path +import pytest + +import scripts.benchmark_routing_algorithms as benchmark from scripts.benchmark_routing_algorithms import ( + BenchmarkArm, BenchmarkConfig, + DirectBaseline, ScenarioDefinition, + compare_to_direct_backend, + materialize_aiperf_input, parse_result, + profile_request_count, write_report, ) @@ -15,6 +25,7 @@ def test_report_combines_scenarios_single_and_aggregate_results(tmp_path) -> None: oha_path = tmp_path / "oha.json" single_path = tmp_path / "profile_export_aiperf.json" + direct_path = tmp_path / "profile_export_aiperf_direct.json" aggregate_path = tmp_path / "profile_export_aiperf_aggregate.json" oha_path.write_text( json.dumps( @@ -40,6 +51,18 @@ def test_report_combines_scenarios_single_and_aggregate_results(tmp_path) -> Non } ) ) + direct_path.write_text( + json.dumps( + { + "error_request_count": {"unit": "requests", "avg": 0}, + "request_count": {"unit": "requests", "avg": 4}, + "request_throughput": {"unit": "requests/sec", "avg": 100.0}, + "request_latency": {"unit": "ms", "p50": 35.0}, + "time_to_first_token": {"unit": "ms", "p50": 12.0, "p99": 30.0}, + "output_token_throughput": {"unit": "tokens/sec", "avg": 700.0}, + } + ) + ) aggregate_path.write_text( json.dumps( { @@ -121,27 +144,51 @@ def test_report_combines_scenarios_single_and_aggregate_results(tmp_path) -> Non }, "routing_overhead": {"count": 22, "total_ms": 50}, } - results = ( - parse_result( - "random", - "switchyard/random", - short, - "fixed", - oha_path, - single_path, - before, - after, - ), - parse_result( - "classifier", - "switchyard/classifier", - failure, - "fixed", - None, - aggregate_path, - before, - after, - ), + results = compare_to_direct_backend( + ( + parse_result( + BenchmarkArm( + "direct-backend", + "mock/weak", + "http://127.0.0.1:8100", + True, + ), + short, + "fixed", + None, + direct_path, + {}, + {}, + ), + parse_result( + BenchmarkArm( + "random", + "switchyard/random", + "http://127.0.0.1:4000", + False, + ), + short, + "fixed", + oha_path, + single_path, + before, + after, + ), + parse_result( + BenchmarkArm( + "classifier", + "switchyard/classifier", + "http://127.0.0.1:4000", + False, + ), + failure, + "fixed", + None, + aggregate_path, + before, + after, + ), + ) ) output_dir = tmp_path / "report" output_dir.mkdir() @@ -155,24 +202,184 @@ def test_report_combines_scenarios_single_and_aggregate_results(tmp_path) -> Non oha_bin="oha", aiperf_bin="aiperf", soak_bin="switchyard-soak", + direct_baseline=DirectBaseline("http://127.0.0.1:8100", "mock/weak"), ) write_report(config, results) report = (output_dir / "report.md").read_text() + assert "## Routing overhead versus the direct backend" in report + assert ( + report.index("## Routing overhead versus the direct backend") + < report.index("## Run details") + < report.index("## Throughput and latency") + ) + assert ( + "| short interactive | random | fixed | +5.00 ms | +3.00 ms | -4.50% | -11.43% |" in report + ) + assert "| short interactive | short baseline |" in report assert "| random | short-interactive | fixed | 120.50 | 95.50 |" in report + assert "## Repeatability" in report assert "## Resilience" in report assert ( "| classifier | failure-pressure | fixed | 50.00% | 1.00%–75.00% | PASS | `{" '"mock/weak":2}` | failures stay bounded |' in report ) rows = list(csv.DictReader((output_dir / "report.csv").open())) - assert rows[0]["selected_model_calls"] == '{"mock/strong":2,"mock/weak":10}' - assert rows[0]["selected_model_share"] == '{"mock/strong":0.1667,"mock/weak":0.8333}' - assert rows[0]["selected_model_errors"] == '{"mock/weak":2}' + rows_by_key = {(row["algorithm"], row["scenario"]): row for row in rows} + random_row = rows_by_key[("random", "short-interactive")] + assert random_row["selected_model_calls"] == '{"mock/strong":2,"mock/weak":10}' + assert random_row["selected_model_share"] == '{"mock/strong":0.1667,"mock/weak":0.8333}' + assert random_row["selected_model_errors"] == '{"mock/weak":2}' payload = json.loads((output_dir / "report.json").read_text()) - assert payload["results"][0]["aiperf_itl_p50_ms"] is None - assert payload["results"][0]["aiperf_output_tokens_per_second_per_user_p50"] == 200.0 - assert payload["results"][1]["aiperf_request_latency_p50_ms"] == 50.0 - assert payload["results"][1]["aiperf_request_throughput_cv"] == 0.03 - assert payload["results"][1]["classifier_latency_avg_ms"] == 4.0 + payload_by_key = {(row["algorithm"], row["scenario"]): row for row in payload["results"]} + direct_row = payload_by_key[("direct-backend", "short-interactive")] + random_payload = payload_by_key[("random", "short-interactive")] + classifier_payload = payload_by_key[("classifier", "failure-pressure")] + assert direct_row["aiperf_itl_p50_ms"] is None + assert direct_row["aiperf_request_throughput_delta_pct"] is None + assert random_payload["aiperf_output_tokens_per_second_per_user_p50"] == 200.0 + assert random_payload["scenario_description"] == "short baseline" + assert classifier_payload["aiperf_request_latency_p50_ms"] == 50.0 + assert classifier_payload["aiperf_request_throughput_cv"] == 0.03 + assert classifier_payload["classifier_latency_avg_ms"] == 4.0 + + with pytest.raises(RuntimeError, match="missing direct baseline for mixed-traffic fixed"): + compare_to_direct_backend((results[0], replace(results[1], scenario="mixed-traffic"))) + + +def test_materialized_sessions_are_unique_and_cover_request_count(tmp_path) -> None: + scenario_path = tmp_path / "short.json" + scenario_path.write_text( + json.dumps( + { + "data": [ + { + "session_id": "short-0", + "payloads": [{"model": "test"}, {"model": "test"}], + } + ] + } + ) + ) + scenario = ScenarioDefinition( + id="short-interactive", + group="core", + description="short baseline", + expected="all requests succeed", + expected_error_rate_min=0.0, + expected_error_rate_max=0.0, + input_file=scenario_path, + load_profiles=(), + ) + + expanded_input = materialize_aiperf_input(scenario, tmp_path / "expanded.json", 5) + expanded_sessions = json.loads(expanded_input.read_text())["data"] + + assert len(expanded_sessions) == 3 + assert len({session["session_id"] for session in expanded_sessions}) == 3 + + +def test_traffic_burst_request_count_integrates_rate_series(tmp_path) -> None: + config = BenchmarkConfig( + base_url="http://127.0.0.1:4000", + models=(("random", "switchyard/random"),), + concurrency=2, + request_count=100, + backend_label="test backend", + output_dir=tmp_path, + oha_bin="oha", + aiperf_bin="aiperf", + soak_bin="switchyard-soak", + ) + profile = { + "kind": "traffic_burst", + "duration_seconds": 30, + "points": [ + {"time_s": 0, "rate_multiplier": 1}, + {"time_s": 10, "rate_multiplier": 1}, + {"time_s": 11, "rate_multiplier": 10}, + {"time_s": 16, "rate_multiplier": 10}, + {"time_s": 17, "rate_multiplier": 1}, + {"time_s": 30, "rate_multiplier": 1}, + ], + } + + assert profile_request_count(config, profile) == 168 + + +def test_aiperf_cells_use_disjoint_artifacts(tmp_path, monkeypatch) -> None: + scenario_path = tmp_path / "short.json" + scenario_path.write_text( + json.dumps( + { + "data": [ + { + "session_id": "short-0", + "payloads": [{"model": "switchyard/random"}], + } + ] + } + ) + ) + scenario = ScenarioDefinition( + id="short-interactive", + group="core", + description="short baseline", + expected="all requests succeed", + expected_error_rate_min=0.0, + expected_error_rate_max=0.0, + input_file=scenario_path, + load_profiles=(), + ) + config = BenchmarkConfig( + base_url="http://127.0.0.1:4000", + models=(("random", "switchyard/random"),), + concurrency=1, + request_count=1, + backend_label="test backend", + output_dir=tmp_path, + oha_bin="oha", + aiperf_bin="aiperf", + soak_bin="switchyard-soak", + profile_runs=1, + ) + arm = BenchmarkArm( + "random", + "switchyard/random", + "http://127.0.0.1:4000", + False, + ) + observed: list[tuple[Path, Path]] = [] + + def fake_run_profile( + _command, log_path: Path, artifact_dir: Path, _timeout_seconds: int + ) -> Path: + observed.append((log_path, artifact_dir)) + artifact_dir.mkdir(parents=True) + export = artifact_dir / "profile_export_aiperf.json" + export.write_text("{}") + return export + + monkeypatch.setattr(benchmark, "run_profile", fake_run_profile) + + first = benchmark.run_aiperf( + config, + arm, + scenario, + {"kind": "fixed"}, + tmp_path, + "algorithm-00-short-interactive-fixed", + ) + second = benchmark.run_aiperf( + config, + arm, + scenario, + {"kind": "fixed"}, + tmp_path, + "algorithm-00-long-context-fixed", + ) + + assert first != second + assert observed[0][0] != observed[1][0] + assert observed[0][1] != observed[1][1] From c3c275f793bfe3562d3ea39a1c7a81c7b84345a2 Mon Sep 17 00:00:00 2001 From: Elyas Mehtabuddin Date: Wed, 19 Aug 2026 13:04:41 -0700 Subject: [PATCH 9/9] feat(soak): add routing overhead plots to reports Signed-off-by: Elyas Mehtabuddin --- crates/switchyard-soak/README.md | 5 +- docs/operations/soak_test.md | 11 +- scripts/benchmark_routing_algorithms.py | 52 +++- scripts/routing_overhead_plot.py | 291 +++++++++++++++++++++++ tests/test_routing_overhead_plot.py | 64 +++++ tests/test_routing_performance_report.py | 8 +- 6 files changed, 419 insertions(+), 12 deletions(-) create mode 100644 scripts/routing_overhead_plot.py create mode 100644 tests/test_routing_overhead_plot.py diff --git a/crates/switchyard-soak/README.md b/crates/switchyard-soak/README.md index f49ffa379..7f739f52f 100644 --- a/crates/switchyard-soak/README.md +++ b/crates/switchyard-soak/README.md @@ -101,8 +101,9 @@ compare the same routes with real model output. oha runs only for the fixed `sho baseline. AIPerf replays every selected session and reports TTFT, ITL, request throughput, output token throughput, multi-run confidence, selected-target calls, classifier calls, and routing overhead. Add `--direct-base-url` and `--direct-model` to compare each route with the same backend -without Switchyard. The operations guide explains how to keep the comparison fair and when to use -the local backend or real models. +without Switchyard. That comparison adds an annotated TTFT and output-token-throughput SVG to the +Markdown report. The operations guide explains how to keep the comparison fair and when to use the +local backend or real models. The script needs the built Rust programs plus installed `oha` and AIPerf commands. `switchyard-soak` itself does not require those extra programs and can run by itself against a diff --git a/docs/operations/soak_test.md b/docs/operations/soak_test.md index e66627109..c0ce76cb4 100644 --- a/docs/operations/soak_test.md +++ b/docs/operations/soak_test.md @@ -77,7 +77,7 @@ cargo build --release -p switchyard-soak Use `scripts/benchmark_routing_algorithms.py` to compare routing algorithms under the same load. The command runs oha and AIPerf sequentially for every model id, then writes `report.md`, -`report.csv`, and `report.json` beside both tools' raw results: +`report.csv`, `report.json`, and `routing-overhead.svg` beside both tools' raw results: ```bash python3.12 scripts/benchmark_routing_algorithms.py \ @@ -112,6 +112,13 @@ all non-resilience request patterns exported by the Rust crate: short and long c outputs, shared prefixes, mixed request sizes, growing conversations, large tool catalogs, tool-call bursts, stage changes, and easy/hard classifier mixes. +When the run includes a direct backend, the Markdown report embeds `routing-overhead.svg` before +the overhead table. The two annotated heatmaps show TTFT and output-token-throughput changes for +every route and workload. Each cell shows the absolute change in milliseconds or tokens per second +and the percent change. Red cells are worse than the direct backend, and blue cells are better. +The script uses only the Python standard library to create the SVG, so the same benchmark command +reproduces the plot without a plotting package. + The Rust crate exports one AIPerf `inputs-json` file per scenario. oha reuses the first exported `short-interactive` payload as a non-streaming fixed body and reports the raw HTTP request rate and latency ceiling. AIPerf replays every selected streaming session and reports request latency, time @@ -209,7 +216,7 @@ The script gives each tool one job: | oha | Measures the non-streaming `short-interactive` HTTP baseline through every route. | | Python route checks | Sends one ordinary Chat Completions request through each configured route before load starts. | | AIPerf | Replays Rust-exported streaming sessions through every route and records LLM, token, response-time, and confidence results. | -| Combined report | Joins scenario, load, oha, AIPerf, and routing-counter metrics in Markdown, CSV, and JSON. It keeps resilience rows separate from throughput rows. | +| Combined report | Joins scenario, load, oha, AIPerf, and routing-counter metrics in Markdown, CSV, JSON, and an overhead plot. It keeps resilience rows separate from throughput rows. | | `switchyard-soak` | Runs the standard scenario set while checking public API variants, server health, metrics, process use, and required results. | `scripts/local_soak_test.toml` exercises `noop`, `random`, `passthrough`, `llm_classifier`, and diff --git a/scripts/benchmark_routing_algorithms.py b/scripts/benchmark_routing_algorithms.py index 635ec181a..9a0a8baa7 100755 --- a/scripts/benchmark_routing_algorithms.py +++ b/scripts/benchmark_routing_algorithms.py @@ -20,8 +20,10 @@ if __package__: from .aiperf_runner import aggregate_exports, run_profile, validate_aiperf_version + from .routing_overhead_plot import OverheadPlotRow, write_overhead_plot else: from aiperf_runner import aggregate_exports, run_profile, validate_aiperf_version + from routing_overhead_plot import OverheadPlotRow, write_overhead_plot MAX_MATERIALIZED_REQUESTS = 1_000_000 MAX_MATERIALIZED_BYTES = 256 * 1024 * 1024 @@ -151,7 +153,9 @@ class BenchmarkResult: aiperf_request_throughput_delta_pct: float | None = None aiperf_request_latency_p50_delta_ms: float | None = None aiperf_ttft_p50_delta_ms: float | None = None + aiperf_ttft_p50_delta_pct: float | None = None aiperf_ttft_p99_delta_ms: float | None = None + aiperf_output_tokens_per_second_delta: float | None = None aiperf_output_tokens_per_second_delta_pct: float | None = None @@ -787,10 +791,18 @@ def compare_to_direct_backend(results: Sequence[BenchmarkResult]) -> list[Benchm result.aiperf_ttft_p50_ms, baseline.aiperf_ttft_p50_ms, ), + aiperf_ttft_p50_delta_pct=_percent_change( + result.aiperf_ttft_p50_ms, + baseline.aiperf_ttft_p50_ms, + ), aiperf_ttft_p99_delta_ms=_difference( result.aiperf_ttft_p99_ms, baseline.aiperf_ttft_p99_ms, ), + aiperf_output_tokens_per_second_delta=_difference( + result.aiperf_output_tokens_per_second, + baseline.aiperf_output_tokens_per_second, + ), aiperf_output_tokens_per_second_delta_pct=_percent_change( result.aiperf_output_tokens_per_second, baseline.aiperf_output_tokens_per_second, @@ -807,7 +819,7 @@ def format_metric(value: float | None) -> str: def format_delta(value: float | None, suffix: str = "") -> str: """Render a signed direct-backend comparison.""" - return "n/a" if value is None else f"{value:+.2f}{suffix}" + return "n/a" if value is None else f"{value:+,.2f}{suffix}" def write_report(config: BenchmarkConfig, results: Sequence[BenchmarkResult]) -> None: @@ -857,6 +869,24 @@ def write_report(config: BenchmarkConfig, results: Sequence[BenchmarkResult]) -> for result in results if not result.bypasses_switchyard and result.scenario_group != "resilience" ] + if overhead_results: + write_overhead_plot( + config.output_dir / "routing-overhead.svg", + [ + OverheadPlotRow( + scenario=result.scenario, + load=result.load_profile, + route=result.algorithm, + ttft_delta_ms=result.aiperf_ttft_p50_delta_ms, + ttft_delta_pct=result.aiperf_ttft_p50_delta_pct, + token_throughput_delta=(result.aiperf_output_tokens_per_second_delta), + token_throughput_delta_pct=( + result.aiperf_output_tokens_per_second_delta_pct + ), + ) + for result in overhead_results + ], + ) lines.extend( ( "## Routing overhead versus the direct backend", @@ -870,14 +900,22 @@ def write_report(config: BenchmarkConfig, results: Sequence[BenchmarkResult]) -> "", ) ) + if overhead_results: + lines.extend( + ( + "![Routing overhead plots](routing-overhead.svg)", + "", + ) + ) if not overhead_results: lines.append("No successful direct-backend comparisons were available.") else: lines.extend( ( - "| Workload | Route | Load | Request p50 | TTFT p50 | Request throughput | " - "Token throughput |", - "|---|---|---|---:|---:|---:|---:|", + "| Workload | Route | Load | Request p50 change (ms) | " + "TTFT p50 change (ms) | TTFT change (%) | Request throughput change (%) | " + "Token throughput change (tokens/s) | Token throughput change (%) |", + "|---|---|---|---:|---:|---:|---:|---:|---:|", ) ) for result in overhead_results: @@ -886,7 +924,9 @@ def write_report(config: BenchmarkConfig, results: Sequence[BenchmarkResult]) -> f"{result.load_profile} | " f"{format_delta(result.aiperf_request_latency_p50_delta_ms, ' ms')} | " f"{format_delta(result.aiperf_ttft_p50_delta_ms, ' ms')} | " + f"{format_delta(result.aiperf_ttft_p50_delta_pct, '%')} | " f"{format_delta(result.aiperf_request_throughput_delta_pct, '%')} | " + f"{format_delta(result.aiperf_output_tokens_per_second_delta, ' tokens/s')} | " f"{format_delta(result.aiperf_output_tokens_per_second_delta_pct, '%')} |" ) lines.extend( @@ -909,9 +949,7 @@ def write_report(config: BenchmarkConfig, results: Sequence[BenchmarkResult]) -> if result.scenario in seen_scenarios: continue seen_scenarios.add(result.scenario) - lines.append( - f"| {result.scenario.replace('-', ' ')} | {result.scenario_description} |" - ) + lines.append(f"| {result.scenario.replace('-', ' ')} | {result.scenario_description} |") lines.extend(("", "## Run details", "", *run_details)) lines.extend( ( diff --git a/scripts/routing_overhead_plot.py b/scripts/routing_overhead_plot.py new file mode 100644 index 000000000..65c6fd2a8 --- /dev/null +++ b/scripts/routing_overhead_plot.py @@ -0,0 +1,291 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Render routing-overhead results as a dependency-free SVG heatmap.""" + +import math +from collections.abc import Sequence +from dataclasses import dataclass +from html import escape +from pathlib import Path +from typing import TypeGuard + + +@dataclass(frozen=True) +class OverheadPlotRow: + """One route and workload comparison against the direct backend.""" + + scenario: str + load: str + route: str + ttft_delta_ms: float | None + ttft_delta_pct: float | None + token_throughput_delta: float | None + token_throughput_delta_pct: float | None + + +@dataclass(frozen=True) +class _Panel: + metric: str + title: str + subtitle: str + primary_suffix: str + primary_values: tuple[float | None, ...] + percent_values: tuple[float | None, ...] + higher_is_better: bool + + +_BACKGROUND = (255, 255, 255) +_BETTER = (42, 111, 174) +_WORSE = (197, 52, 52) +_NEUTRAL = "#f3f5f7" + + +def _finite(value: float | None) -> TypeGuard[float]: + return value is not None and math.isfinite(value) + + +def _fill(value: float | None, maximum: float, higher_is_better: bool) -> str: + if not _finite(value) or value == 0: + return _NEUTRAL + worse = value < 0 if higher_is_better else value > 0 + target = _WORSE if worse else _BETTER + intensity = 0.16 + 0.64 * min(abs(value) / maximum, 1.0) + channels = ( + round(background + (foreground - background) * intensity) + for background, foreground in zip(_BACKGROUND, target, strict=True) + ) + red, green, blue = channels + return f"#{red:02x}{green:02x}{blue:02x}" + + +def _format_value(value: float | None, suffix: str) -> str: + if not _finite(value): + return "n/a" + return f"{value:+,.2f}{suffix}" + + +def _text( + x: float, + y: float, + value: str, + anchor: str = "start", + size: int = 13, + weight: int = 400, + fill: str = "#1d2733", +) -> str: + return ( + f'{escape(value)}' + ) + + +def write_overhead_plot(path: Path, rows: Sequence[OverheadPlotRow]) -> None: + """Write an annotated SVG overview for every supplied route and workload.""" + if not rows: + raise ValueError("an overhead plot requires at least one row") + + workloads = list(dict.fromkeys((row.scenario, row.load) for row in rows)) + routes = list(dict.fromkeys(row.route for row in rows)) + by_key: dict[tuple[str, str, str], OverheadPlotRow] = {} + for row in rows: + key = (row.scenario, row.load, row.route) + if key in by_key: + raise RuntimeError( + f"duplicate overhead plot row for {row.scenario}, {row.load}, and {row.route}" + ) + by_key[key] = row + + ordered_rows = [ + by_key.get((scenario, load, route)) for scenario, load in workloads for route in routes + ] + panels = ( + _Panel( + "ttft_p50_delta_ms", + "TTFT p50 overhead", + "Each cell shows the absolute change in ms and the percent change; lower is better", + " ms", + tuple(row.ttft_delta_ms if row is not None else None for row in ordered_rows), + tuple(row.ttft_delta_pct if row is not None else None for row in ordered_rows), + False, + ), + _Panel( + "output_tokens_per_second_delta", + "Output token throughput overhead", + "Each cell shows the absolute change in tokens/s and the percent change; higher is better", + " tokens/s", + tuple(row.token_throughput_delta if row is not None else None for row in ordered_rows), + tuple( + row.token_throughput_delta_pct if row is not None else None for row in ordered_rows + ), + True, + ), + ) + + label_width = 250 + cell_width = 170 + row_height = 50 + panel_header_height = 76 + panel_gap = 52 + top = 112 + right = 24 + panel_height = panel_header_height + len(workloads) * row_height + width = label_width + len(routes) * cell_width + right + height = top + len(panels) * panel_height + panel_gap + 28 + center = width / 2 + legend_x = center - 260 + + svg = [ + ( + f'' + ), + 'Routing overhead versus the direct backend', + ( + 'Two annotated heatmaps compare median ' + "time to first token in milliseconds and output token throughput in tokens per " + "second for each route and workload. Every cell also gives the percent change. Red " + "cells are worse than the direct backend and blue cells are better. Darker cells " + "show a larger percent change within each heatmap." + ), + f'', + _text( + center, + 31, + "Routing overhead versus the direct backend", + anchor="middle", + size=22, + weight=700, + ), + _text( + center, + 55, + "Measures Switchyard overhead when both arms use the same deployment, model, and settings", + anchor="middle", + size=13, + fill="#596777", + ), + f'', + _text(legend_x + 24, 84, "worse", size=12, fill="#596777"), + f'', + _text(legend_x + 104, 84, "better", size=12, fill="#596777"), + f'', + _text( + legend_x + 187, + 84, + "no change / unavailable; darker = larger % change", + size=12, + fill="#596777", + ), + ] + + for panel_index, panel in enumerate(panels): + panel_y = top + panel_index * (panel_height + panel_gap) + svg.append(_text(center, panel_y + 18, panel.title, anchor="middle", size=16, weight=650)) + svg.append( + _text( + center, + panel_y + 39, + panel.subtitle, + anchor="middle", + size=12, + fill="#596777", + ) + ) + svg.append( + _text( + label_width / 2, + panel_y + 66, + "Workload / load", + anchor="middle", + size=12, + weight=600, + ) + ) + for route_index, route in enumerate(routes): + cell_x = label_width + route_index * cell_width + svg.append( + _text( + cell_x + cell_width / 2, + panel_y + 66, + route.replace("-", " ").replace("_", " "), + anchor="middle", + size=12, + weight=600, + ) + ) + + finite_values = [abs(value) for value in panel.percent_values if _finite(value)] + maximum = max(finite_values, default=1.0) or 1.0 + row_y = panel_y + panel_header_height + for workload_index, (scenario, load) in enumerate(workloads): + y = row_y + workload_index * row_height + svg.append( + _text( + label_width / 2, + y + 21, + scenario.replace("-", " "), + anchor="middle", + size=12, + fill="#344150", + ) + ) + svg.append( + _text( + label_width / 2, + y + 38, + load.replace("-", " "), + anchor="middle", + size=11, + fill="#697786", + ) + ) + for route_index in range(len(routes)): + value_index = workload_index * len(routes) + route_index + primary_value = panel.primary_values[value_index] + percent_value = panel.percent_values[value_index] + x = label_width + route_index * cell_width + cell_title = ( + f"{scenario}, {load}, {routes[route_index]}: " + f"{_format_value(primary_value, panel.primary_suffix)} " + f"({_format_value(percent_value, '%')})" + ) + svg.append( + f'' + ) + svg.append(f"{escape(cell_title)}") + svg.append( + f'' + ) + svg.append( + _text( + x + cell_width / 2, + y + 21, + _format_value(primary_value, panel.primary_suffix), + anchor="middle", + size=12, + weight=600, + ) + ) + svg.append( + _text( + x + cell_width / 2, + y + 38, + f"({_format_value(percent_value, '%')})", + anchor="middle", + size=11, + fill="#44515f", + ) + ) + svg.append("") + + svg.append("") + path.write_text("\n".join(svg) + "\n", encoding="utf-8") diff --git a/tests/test_routing_overhead_plot.py b/tests/test_routing_overhead_plot.py new file mode 100644 index 000000000..9a7ddf42e --- /dev/null +++ b/tests/test_routing_overhead_plot.py @@ -0,0 +1,64 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import xml.etree.ElementTree as ET + +from scripts.routing_overhead_plot import OverheadPlotRow, write_overhead_plot + + +def test_overhead_plot_preserves_matrix_and_unavailable_values(tmp_path) -> None: + path = tmp_path / "routing-overhead.svg" + rows = ( + OverheadPlotRow("short-", "fixed", "random", 3.0, 25.0, -80.0, -11.43), + OverheadPlotRow("short-", "fixed", "stage&router", None, None, -14.0, -2.0), + OverheadPlotRow("long-context", "traffic-burst", "random", 1.25, 3.0, None, None), + OverheadPlotRow("long-context", "traffic-burst", "stage&router", -0.5, -1.0, 7.0, 1.0), + ) + + write_overhead_plot(path, rows) + + root = ET.fromstring(path.read_text()) + namespace = {"svg": "http://www.w3.org/2000/svg"} + cells = root.findall(".//svg:g", namespace) + text_nodes = root.findall(".//svg:text", namespace) + values = { + ( + cell.attrib["data-scenario"], + cell.attrib["data-load"], + cell.attrib["data-route"], + cell.attrib["data-metric"], + ): " ".join("".join(cell.itertext()).split()) + for cell in cells + } + assert len(values) == 8 + assert ( + values[ + ( + "short-", + "fixed", + "random", + "output_tokens_per_second_delta", + ) + ] + == "short-, fixed, random: -80.00 tokens/s (-11.43%) " + "-80.00 tokens/s (-11.43%)" + ) + assert ( + "n/a" + in values[("long-context", "traffic-burst", "random", "output_tokens_per_second_delta")] + ) + assert "n/a" in values[("short-", "fixed", "stage&router", "ttft_p50_delta_ms")] + centered_labels = { + node.text: node.attrib["text-anchor"] + for node in text_nodes + if node.text in {"Routing overhead versus the direct backend", "short "} + } + assert centered_labels == { + "Routing overhead versus the direct backend": "middle", + "short ": "middle", + } + assert all( + text.attrib["text-anchor"] == "middle" + for cell in cells + for text in cell.findall("svg:text", namespace) + ) diff --git a/tests/test_routing_performance_report.py b/tests/test_routing_performance_report.py index 2d9c31f0c..445357cc1 100644 --- a/tests/test_routing_performance_report.py +++ b/tests/test_routing_performance_report.py @@ -209,13 +209,15 @@ def test_report_combines_scenarios_single_and_aggregate_results(tmp_path) -> Non report = (output_dir / "report.md").read_text() assert "## Routing overhead versus the direct backend" in report + assert "![Routing overhead plots](routing-overhead.svg)" in report assert ( report.index("## Routing overhead versus the direct backend") < report.index("## Run details") < report.index("## Throughput and latency") ) assert ( - "| short interactive | random | fixed | +5.00 ms | +3.00 ms | -4.50% | -11.43% |" in report + "| short interactive | random | fixed | +5.00 ms | +3.00 ms | +25.00% | " + "-4.50% | -80.00 tokens/s | -11.43% |" in report ) assert "| short interactive | short baseline |" in report assert "| random | short-interactive | fixed | 120.50 | 95.50 |" in report @@ -239,11 +241,15 @@ def test_report_combines_scenarios_single_and_aggregate_results(tmp_path) -> Non assert direct_row["aiperf_itl_p50_ms"] is None assert direct_row["aiperf_request_throughput_delta_pct"] is None assert random_payload["aiperf_output_tokens_per_second_per_user_p50"] == 200.0 + assert random_payload["aiperf_ttft_p50_delta_pct"] == 25.0 + assert random_payload["aiperf_output_tokens_per_second_delta"] == -80.0 assert random_payload["scenario_description"] == "short baseline" assert classifier_payload["aiperf_request_latency_p50_ms"] == 50.0 assert classifier_payload["aiperf_request_throughput_cv"] == 0.03 assert classifier_payload["classifier_latency_avg_ms"] == 4.0 + assert (output_dir / "routing-overhead.svg").is_file() + with pytest.raises(RuntimeError, match="missing direct baseline for mixed-traffic fixed"): compare_to_direct_backend((results[0], replace(results[1], scenario="mixed-traffic")))