diff --git a/Cargo.lock b/Cargo.lock index 405dda9da..83cfc8ede 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2301,6 +2301,7 @@ dependencies = [ "opentelemetry_sdk", "parking_lot", "reqwest", + "serde", "serde_json", "switchyard-libsy", "switchyard-protocol", diff --git a/crates/libsy-llm-client/Cargo.toml b/crates/libsy-llm-client/Cargo.toml index 9a9ca6e0a..a347ed7cc 100644 --- a/crates/libsy-llm-client/Cargo.toml +++ b/crates/libsy-llm-client/Cargo.toml @@ -29,6 +29,7 @@ parking_lot.workspace = true http.workspace = true httpdate.workspace = true serde_json.workspace = true +serde.workspace = true tokio.workspace = true tracing.workspace = true tracing-opentelemetry.workspace = true diff --git a/crates/libsy-llm-client/README.md b/crates/libsy-llm-client/README.md index b62da05e1..d5e7f1175 100644 --- a/crates/libsy-llm-client/README.md +++ b/crates/libsy-llm-client/README.md @@ -246,6 +246,9 @@ fn build_multi_format_client( transport failures, timeouts, HTTP 408/429, and 5xx responses. Buffered body transport failures are retried; streaming body failures are not replayed after the response has been returned. +- `ModelConfig::with_responses_reasoning` controls reasoning-item replay for a + Responses model. `PreserveEncrypted` keeps signed provider state without + plaintext; `Drop` removes reasoning while retaining messages and tool history. Retries replay the same upstream request to the same model. Each candidate's `max_retries` budget is exhausted before candidate fallback advances to the next diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rs index d58508ca5..616871801 100644 --- a/crates/libsy-llm-client/src/client.rs +++ b/crates/libsy-llm-client/src/client.rs @@ -62,6 +62,7 @@ pub struct ModelConfig { model_name: ModelId, default_backend: Backend, other_backends: Option>, + responses_reasoning: crate::ResponsesReasoningPolicy, } impl ModelConfig { @@ -76,8 +77,16 @@ impl ModelConfig { model_name: model_name.into(), default_backend, other_backends, + responses_reasoning: crate::ResponsesReasoningPolicy::default(), } } + + /// Sets how Responses reasoning items are replayed to this model. + #[must_use] + pub fn with_responses_reasoning(mut self, policy: crate::ResponsesReasoningPolicy) -> Self { + self.responses_reasoning = policy; + self + } } /// A client that dispatches neutral-IR requests to per-model HTTP backends. @@ -221,6 +230,13 @@ impl TranslatingLlmClient { strip_anthropic_incompatible_fields(&mut body); strip_unsigned_thinking_blocks(&mut body); } + if matches!(backend, Backend::OpenAiResponses(_)) { + self.model_to_config + .get(model) + .map(|config| config.responses_reasoning) + .unwrap_or_default() + .normalize(&mut body); + } merge_extra_body(&mut body, backend.extra_body()); if matches!(backend, Backend::Anthropic(_)) { enable_anthropic_prompt_caching(&mut body); @@ -871,6 +887,21 @@ mod tests { )] } + fn responses_map(base_url: &str) -> Vec { + vec![ModelConfig::new( + "gpt", + Backend::OpenAiResponses(config(base_url)), + None, + )] + } + + fn local_responses_map(base_url: &str) -> Vec { + vec![ + ModelConfig::new("local", Backend::OpenAiResponses(config(base_url)), None) + .with_responses_reasoning(crate::ResponsesReasoningPolicy::Drop), + ] + } + fn chat_map_with_retries(base_url: &str, max_retries: u32) -> Vec { vec![ModelConfig::new( "gpt", @@ -1368,6 +1399,149 @@ mod tests { Ok(()) } + // Local Responses backends can emit plaintext reasoning that strict upstreams cannot replay. + // Conversely, local backends require signed reasoning replayed from a strict + // upstream to retain `content` as an array, even though it must be empty. + #[tokio::test] + async fn responses_requests_drop_unsigned_reasoning_items() + -> std::result::Result<(), Box> { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/v1/responses")) + .and(|request: &wiremock::Request| { + let body: Value = serde_json::from_slice(&request.body).unwrap_or(Value::Null); + let Some(input) = body.get("input").and_then(Value::as_array) else { + return false; + }; + let reasoning: Vec<&Value> = input + .iter() + .filter(|item| item.get("type").and_then(Value::as_str) == Some("reasoning")) + .collect(); + reasoning.len() == 1 + && reasoning[0] + .get("encrypted_content") + .and_then(Value::as_str) + == Some("encrypted") + && reasoning[0].get("content") == Some(&json!([])) + && input.iter().any(|item| { + item.get("type").and_then(Value::as_str) == Some("function_call") + }) + && input.iter().any(|item| { + item.get("type").and_then(Value::as_str) == Some("function_call_output") + }) + }) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "id": "resp_1", + "object": "response", + "model": "gpt", + "status": "completed", + "output": [{ + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "ok"}] + }], + "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2} + }))) + .mount(&server) + .await; + + let client = TranslatingLlmClient::new(&responses_map(&format!("{}/v1", server.uri())))?; + let raw = json!({ + "model": "client-facing", + "input": [ + {"type": "message", "role": "user", "content": "inspect the repo"}, + { + "type": "reasoning", + "content": [{"type": "reasoning_text", "text": "private local reasoning"}], + "encrypted_content": "" + }, + {"type": "function_call", "call_id": "call_1", "name": "shell", "arguments": "{}"}, + {"type": "function_call_output", "call_id": "call_1", "output": "ok"}, + { + "type": "reasoning", + "content": [{"type": "reasoning_text", "text": "must not be replayed"}], + "encrypted_content": "encrypted" + } + ] + }); + + client + .call_rewrite_model_raw( + raw, + None, + Some(&ModelId::from("gpt")), + WireFormat::OpenAiResponses, + ) + .await?; + Ok(()) + } + + // Encrypted hosted reasoning is opaque to a local Responses backend. Dropping + // it avoids llama.cpp rejecting a missing or empty `content` array while + // retaining the conversation and tool-call history it can consume. + #[tokio::test] + async fn local_responses_requests_drop_encrypted_reasoning_items() + -> std::result::Result<(), Box> { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/v1/responses")) + .and(|request: &wiremock::Request| { + let body: Value = serde_json::from_slice(&request.body).unwrap_or(Value::Null); + let Some(input) = body.get("input").and_then(Value::as_array) else { + return false; + }; + input + .iter() + .all(|item| item.get("type").and_then(Value::as_str) != Some("reasoning")) + && input.iter().any(|item| { + item.get("type").and_then(Value::as_str) == Some("function_call") + }) + && input.iter().any(|item| { + item.get("type").and_then(Value::as_str) == Some("function_call_output") + }) + }) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "id": "resp_local", + "object": "response", + "model": "local", + "status": "completed", + "output": [{ + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "ok"}] + }], + "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2} + }))) + .mount(&server) + .await; + + let client = + TranslatingLlmClient::new(&local_responses_map(&format!("{}/v1", server.uri())))?; + let raw = json!({ + "model": "client-facing", + "input": [ + {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "inspect"}]}, + { + "type": "reasoning", + "encrypted_content": "opaque-provider-reasoning" + }, + {"type": "function_call", "call_id": "call_1", "name": "shell", "arguments": "{}"}, + {"type": "function_call_output", "call_id": "call_1", "output": "ok"}, + {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "continue"}]} + ] + }); + + client + .call_rewrite_model_raw( + raw, + None, + Some(&ModelId::from("local")), + WireFormat::OpenAiResponses, + ) + .await?; + Ok(()) + } + // A router can serve earlier turns from an OpenAI target and later turns from // an Anthropic one, so the Anthropic leg must drop OpenAI-only fields the // caller keeps sending or the upstream rejects the whole request. diff --git a/crates/libsy-llm-client/src/lib.rs b/crates/libsy-llm-client/src/lib.rs index 059675bbe..dd45d865c 100644 --- a/crates/libsy-llm-client/src/lib.rs +++ b/crates/libsy-llm-client/src/lib.rs @@ -23,6 +23,7 @@ pub mod metrics; mod observability; mod observation; pub mod raw; +mod responses_reasoning; pub mod run; pub use backend::{Backend, DEFAULT_MAX_RETRIES, HttpBackendConfig}; @@ -30,6 +31,7 @@ pub use client::{ModelConfig, TranslatingLlmClient}; pub use error::{LlmClientError, Result}; pub use observation::{LlmCallObservation, RunObservation, RunObserver}; pub use raw::RawResponse; +pub use responses_reasoning::ResponsesReasoningPolicy; pub use run::{ClientRouter, run}; pub use switchyard_translation::RawEventStream; diff --git a/crates/libsy-llm-client/src/responses_reasoning.rs b/crates/libsy-llm-client/src/responses_reasoning.rs new file mode 100644 index 000000000..fba6e868a --- /dev/null +++ b/crates/libsy-llm-client/src/responses_reasoning.rs @@ -0,0 +1,116 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Target-specific replay policy for OpenAI Responses reasoning items. + +use serde::Deserialize; +use serde_json::Value; + +/// Controls which Responses reasoning items are replayed to an upstream. +#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, PartialEq)] +#[serde(rename_all = "snake_case")] +pub enum ResponsesReasoningPolicy { + /// Preserve provider-encrypted reasoning but remove plaintext reasoning. + /// + /// This is the safe default for strict hosted Responses providers. + #[default] + PreserveEncrypted, + /// Drop all reasoning items while preserving messages and tool-call history. + /// + /// Use this for local Responses-compatible servers that cannot consume + /// another provider's encrypted reasoning representation. + Drop, +} + +impl ResponsesReasoningPolicy { + /// Normalizes a Responses request body for this replay policy. + pub(crate) fn normalize(self, body: &mut Value) { + let Some(Value::Array(input)) = body.get_mut("input") else { + return; + }; + input.retain_mut(|item| self.normalize_item(item)); + } + + fn normalize_item(self, item: &mut Value) -> bool { + let Some(object) = item.as_object_mut() else { + return true; + }; + if object.get("type").and_then(Value::as_str) != Some("reasoning") { + return true; + } + + let signed = matches!( + object.get("encrypted_content").and_then(Value::as_str), + Some(encrypted_content) if !encrypted_content.is_empty() + ); + if self == Self::PreserveEncrypted && signed { + object.insert("content".to_string(), Value::Array(Vec::new())); + true + } else { + false + } + } +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + fn mixed_history() -> Value { + json!({ + "input": [ + {"type": "message", "role": "user", "content": []}, + { + "type": "reasoning", + "content": [{"type": "reasoning_text", "text": "plaintext"}], + "encrypted_content": "" + }, + {"type": "function_call", "call_id": "call_1"}, + {"type": "function_call_output", "call_id": "call_1", "output": "ok"}, + { + "type": "reasoning", + "content": [{"type": "reasoning_text", "text": "must be removed"}], + "encrypted_content": "encrypted" + } + ] + }) + } + + #[test] + fn preserve_encrypted_drops_unsigned_and_clears_plaintext() { + let mut body = mixed_history(); + ResponsesReasoningPolicy::PreserveEncrypted.normalize(&mut body); + + let input = body["input"].as_array().expect("input array"); + let reasoning: Vec<&Value> = input + .iter() + .filter(|item| item["type"] == "reasoning") + .collect(); + assert_eq!(reasoning.len(), 1); + assert_eq!(reasoning[0]["encrypted_content"], "encrypted"); + assert_eq!(reasoning[0]["content"], json!([])); + assert!(input.iter().any(|item| item["type"] == "function_call")); + assert!( + input + .iter() + .any(|item| item["type"] == "function_call_output") + ); + } + + #[test] + fn drop_removes_all_reasoning_and_keeps_tool_history() { + let mut body = mixed_history(); + ResponsesReasoningPolicy::Drop.normalize(&mut body); + + let input = body["input"].as_array().expect("input array"); + assert!(input.iter().all(|item| item["type"] != "reasoning")); + assert!(input.iter().any(|item| item["type"] == "function_call")); + assert!( + input + .iter() + .any(|item| item["type"] == "function_call_output") + ); + } +} diff --git a/crates/switchyard-server/README.md b/crates/switchyard-server/README.md index cf107fecc..3086feed2 100644 --- a/crates/switchyard-server/README.md +++ b/crates/switchyard-server/README.md @@ -85,6 +85,10 @@ Target-level `extra_body` values are shallow-merged into the upstream request wh the request does not already contain that key. `max_retries` defaults to `2` and applies to transport failures, timeouts, HTTP 408/429, and 5xx responses. +For an `openai_responses` client, `responses_reasoning` defaults to +`preserve_encrypted`. Set it to `drop` for a local compatible server that cannot +replay another provider's encrypted reasoning items; messages and tool history +are retained. Random-route `weights` are relative, follow target order, and do not need to sum to one. Omit them for equal weighting. The optional `seed` reproduces the selection sequence for the same call order. diff --git a/crates/switchyard-server/src/config.rs b/crates/switchyard-server/src/config.rs index a78695cd5..71f3129c9 100644 --- a/crates/switchyard-server/src/config.rs +++ b/crates/switchyard-server/src/config.rs @@ -18,7 +18,7 @@ use serde::Deserialize; use serde_json::Value; use switchyard_llm_client::{ Backend, ClientRouter, DEFAULT_MAX_RETRIES, HttpBackendConfig, ModelConfig, - TranslatingLlmClient, + ResponsesReasoningPolicy, TranslatingLlmClient, }; use switchyard_protocol::{ModelId, RoutedLlmClient}; @@ -135,11 +135,14 @@ impl ServerConfig { let model_configs = models_by_client .get_mut(&target.llm_client) .ok_or_else(|| ServerError::new("validated llm client was not initialized"))?; - model_configs.push(ModelConfig::new( - target.id.clone(), - build_backend(&target.llm_client, client_config, &target.extra_body)?, - None, - )); + model_configs.push( + ModelConfig::new( + target.id.clone(), + build_backend(&target.llm_client, client_config, &target.extra_body)?, + None, + ) + .with_responses_reasoning(client_config.responses_reasoning.unwrap_or_default()), + ); } let mut clients = BTreeMap::new(); @@ -256,6 +259,7 @@ struct LlmClientConfig { extra_headers: BTreeMap, #[serde(default = "default_max_retries")] max_retries: u32, + responses_reasoning: Option, } #[derive(Debug, Deserialize)] @@ -267,7 +271,7 @@ struct TargetConfig { extra_body: BTreeMap, } -#[derive(Clone, Copy, Debug, Deserialize)] +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq)] enum ClientFormat { #[serde(rename = "openai_chat")] OpenAiChat, @@ -875,6 +879,11 @@ fn build_backend( "llm client {client_name} cannot set both forward_auth and api_key_env" ))); } + if config.responses_reasoning.is_some() && config.format != ClientFormat::OpenAiResponses { + return Err(ServerError::new(format!( + "llm client {client_name} responses_reasoning is only valid for openai_responses" + ))); + } let api_key = config .api_key_env .as_deref() @@ -1655,6 +1664,48 @@ target = "azure" Ok(()) } + #[test] + fn responses_reasoning_defaults_and_accepts_drop() -> ServerResult<()> { + let default: ServerConfig = toml::from_str(VALID_CONFIG) + .map_err(|error| ServerError::new(format!("failed to parse config: {error}")))?; + let responses = default + .llm_clients + .get("responses") + .ok_or_else(|| ServerError::new("responses llm client is missing"))?; + assert_eq!(responses.responses_reasoning, None); + + let configured = VALID_CONFIG.replacen( + "[llm_clients.responses]\nformat = \"openai_responses\"\nbase_url = \"https://example.test/v1\"", + "[llm_clients.responses]\nformat = \"openai_responses\"\nbase_url = \"https://example.test/v1\"\nresponses_reasoning = \"drop\"", + 1, + ); + let config: ServerConfig = toml::from_str(&configured) + .map_err(|error| ServerError::new(format!("failed to parse config: {error}")))?; + let responses = config + .llm_clients + .get("responses") + .ok_or_else(|| ServerError::new("responses llm client is missing"))?; + assert_eq!( + responses.responses_reasoning, + Some(ResponsesReasoningPolicy::Drop) + ); + server_state_from_toml(&configured)?; + Ok(()) + } + + #[test] + fn responses_reasoning_rejects_other_formats() { + let invalid = VALID_CONFIG.replacen( + "format = \"openai_chat\"", + "format = \"openai_chat\"\nresponses_reasoning = \"drop\"", + 1, + ); + assert!( + error_message(&invalid) + .contains("responses_reasoning is only valid for openai_responses") + ); + } + #[test] fn rejects_headers_that_switchyard_sets() { let cases = [ diff --git a/docs/getting_started.md b/docs/getting_started.md index 58b59efe6..e31d7cb93 100644 --- a/docs/getting_started.md +++ b/docs/getting_started.md @@ -161,6 +161,9 @@ caller's credential to that upstream. OpenAI clients forward `authorization`, `authorization` or `x-api-key`. Enable this only for an upstream that should receive the caller's login. The server rejects a forwarding route called through the other provider's API. +For a local `openai_responses` server that cannot replay provider-encrypted +reasoning, set `responses_reasoning = "drop"`; hosted Responses clients default +to `preserve_encrypted`. ### Run the server diff --git a/docs/reference/toml_schema.md b/docs/reference/toml_schema.md index 68d01bc54..1956e2366 100644 --- a/docs/reference/toml_schema.md +++ b/docs/reference/toml_schema.md @@ -48,6 +48,7 @@ route reaches no upstream. A file without a `[targets]` table is rejected with | `forward_auth` | No | `false` | Forward the caller's provider credential to this upstream. | | `extra_headers` | No | `{}` | Custom HTTP headers sent to the model server. Set credentials with `api_key_env` or `forward_auth`; the server rejects headers owned by the selected auth mode. Header names are case-insensitive. | | `max_retries` | No | `2` | Retry budget, `0`–`10`. | +| `responses_reasoning` | No | `preserve_encrypted` | Reasoning replay policy for `openai_responses`: `preserve_encrypted` or `drop`. Rejected for other formats. | The TOML never contains the secret itself. `api_key_env` names a variable that must exist and be non-empty when the server loads. @@ -76,6 +77,23 @@ server rejects an Anthropic forwarding route called through an OpenAI endpoint, or an OpenAI forwarding route called through an Anthropic endpoint, before it calls an upstream. +Responses providers do not share one reasoning representation. Strict hosted +providers can replay signed `encrypted_content`, while local compatible servers +may reject those opaque items. The default `responses_reasoning = +"preserve_encrypted"` drops unsigned plaintext reasoning and keeps signed +reasoning without plaintext. Configure a local client explicitly when it cannot +consume encrypted reasoning: + +```toml +[llm_clients.local] +format = "openai_responses" +base_url = "http://127.0.0.1:8080/v1" +responses_reasoning = "drop" +``` + +The `drop` policy removes reasoning items only; messages, function calls, and +function-call outputs remain in order. + ## `[targets.]` | Key | Required | Default | Meaning |