From 30c648c2e88f3b5f523ca62deff988e11f4fa95d Mon Sep 17 00:00:00 2001 From: Sumanth Chandrupatla Date: Tue, 18 Aug 2026 21:11:22 -0500 Subject: [PATCH 1/5] fix: sanitize unsigned Responses reasoning handoffs Signed-off-by: Sumanth Chandrupatla --- crates/libsy-llm-client/src/client.rs | 109 ++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rs index d58508ca5..5924d5db0 100644 --- a/crates/libsy-llm-client/src/client.rs +++ b/crates/libsy-llm-client/src/client.rs @@ -221,6 +221,9 @@ impl TranslatingLlmClient { strip_anthropic_incompatible_fields(&mut body); strip_unsigned_thinking_blocks(&mut body); } + if matches!(backend, Backend::OpenAiResponses(_)) { + strip_unsigned_responses_reasoning(&mut body); + } merge_extra_body(&mut body, backend.extra_body()); if matches!(backend, Backend::Anthropic(_)) { enable_anthropic_prompt_caching(&mut body); @@ -676,6 +679,29 @@ fn set_json_model(body: &mut Value, model: &str) { } } +// Removes plaintext reasoning items that strict Responses backends cannot replay. +fn strip_unsigned_responses_reasoning(body: &mut Value) { + let Some(Value::Array(input)) = body.get_mut("input") else { + return; + }; + input.retain_mut(|item| { + let Some(object) = item.as_object_mut() else { + return true; + }; + if object.get("type").and_then(Value::as_str) != Some("reasoning") { + return true; + } + let signed = matches!( + object.get("encrypted_content").and_then(Value::as_str), + Some(encrypted_content) if !encrypted_content.is_empty() + ); + if signed { + object.remove("content"); + } + signed + }); +} + // Drops fields accepted by OpenAI-like APIs but rejected by Anthropic Messages. // // A router can serve earlier turns of a session from an OpenAI-format target and @@ -871,6 +897,14 @@ mod tests { )] } + fn responses_map(base_url: &str) -> Vec { + vec![ModelConfig::new( + "gpt", + Backend::OpenAiResponses(config(base_url)), + None, + )] + } + fn chat_map_with_retries(base_url: &str, max_retries: u32) -> Vec { vec![ModelConfig::new( "gpt", @@ -1368,6 +1402,81 @@ mod tests { Ok(()) } + // Local Responses backends can emit plaintext reasoning that strict upstreams cannot replay. + #[tokio::test] + async fn responses_requests_drop_unsigned_reasoning_items() + -> std::result::Result<(), Box> { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/v1/responses")) + .and(|request: &wiremock::Request| { + let body: Value = serde_json::from_slice(&request.body).unwrap_or(Value::Null); + let Some(input) = body.get("input").and_then(Value::as_array) else { + return false; + }; + let reasoning: Vec<&Value> = input + .iter() + .filter(|item| item.get("type").and_then(Value::as_str) == Some("reasoning")) + .collect(); + reasoning.len() == 1 + && reasoning[0] + .get("encrypted_content") + .and_then(Value::as_str) + == Some("encrypted") + && reasoning[0].get("content").is_none() + && input.iter().any(|item| { + item.get("type").and_then(Value::as_str) == Some("function_call") + }) + && input.iter().any(|item| { + item.get("type").and_then(Value::as_str) == Some("function_call_output") + }) + }) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "id": "resp_1", + "object": "response", + "model": "gpt", + "status": "completed", + "output": [{ + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "ok"}] + }], + "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2} + }))) + .mount(&server) + .await; + + let client = TranslatingLlmClient::new(&responses_map(&format!("{}/v1", server.uri())))?; + let raw = json!({ + "model": "client-facing", + "input": [ + {"type": "message", "role": "user", "content": "inspect the repo"}, + { + "type": "reasoning", + "content": [{"type": "reasoning_text", "text": "private local reasoning"}], + "encrypted_content": "" + }, + {"type": "function_call", "call_id": "call_1", "name": "shell", "arguments": "{}"}, + {"type": "function_call_output", "call_id": "call_1", "output": "ok"}, + { + "type": "reasoning", + "content": [{"type": "reasoning_text", "text": "must not be replayed"}], + "encrypted_content": "encrypted" + } + ] + }); + + client + .call_rewrite_model_raw( + raw, + None, + Some(&ModelId::from("gpt")), + WireFormat::OpenAiResponses, + ) + .await?; + Ok(()) + } + // A router can serve earlier turns from an OpenAI target and later turns from // an Anthropic one, so the Anthropic leg must drop OpenAI-only fields the // caller keeps sending or the upstream rejects the whole request. From 1fd39ec2239392ef2a0c3bf303b0f138288d2ae6 Mon Sep 17 00:00:00 2001 From: Sumanth Chandrupatla Date: Wed, 19 Aug 2026 07:15:28 -0500 Subject: [PATCH 2/5] fix: preserve reasoning content arrays across handoffs Signed-off-by: Sumanth Chandrupatla --- crates/libsy-llm-client/src/client.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rs index 5924d5db0..b5c139e21 100644 --- a/crates/libsy-llm-client/src/client.rs +++ b/crates/libsy-llm-client/src/client.rs @@ -680,6 +680,9 @@ fn set_json_model(body: &mut Value, model: &str) { } // Removes plaintext reasoning items that strict Responses backends cannot replay. +// Signed reasoning keeps an empty content array: strict OpenAI requires that the +// array contain no plaintext, while OpenAI-compatible backends such as llama.cpp +// require the field itself to remain an array when replaying the item. fn strip_unsigned_responses_reasoning(body: &mut Value) { let Some(Value::Array(input)) = body.get_mut("input") else { return; @@ -696,7 +699,7 @@ fn strip_unsigned_responses_reasoning(body: &mut Value) { Some(encrypted_content) if !encrypted_content.is_empty() ); if signed { - object.remove("content"); + object.insert("content".to_string(), Value::Array(Vec::new())); } signed }); @@ -1403,6 +1406,8 @@ mod tests { } // Local Responses backends can emit plaintext reasoning that strict upstreams cannot replay. + // Conversely, local backends require signed reasoning replayed from a strict + // upstream to retain `content` as an array, even though it must be empty. #[tokio::test] async fn responses_requests_drop_unsigned_reasoning_items() -> std::result::Result<(), Box> { @@ -1423,7 +1428,7 @@ mod tests { .get("encrypted_content") .and_then(Value::as_str) == Some("encrypted") - && reasoning[0].get("content").is_none() + && reasoning[0].get("content") == Some(&json!([])) && input.iter().any(|item| { item.get("type").and_then(Value::as_str) == Some("function_call") }) From ab1def4174ade94321e6b4cbd9fb6e9c63a8a9f7 Mon Sep 17 00:00:00 2001 From: Sumanth Chandrupatla Date: Wed, 19 Aug 2026 07:18:21 -0500 Subject: [PATCH 3/5] fix: drop encrypted reasoning for local backends Signed-off-by: Sumanth Chandrupatla --- crates/libsy-llm-client/src/backend.rs | 11 +++ crates/libsy-llm-client/src/client.rs | 93 +++++++++++++++++++++++--- 2 files changed, 96 insertions(+), 8 deletions(-) diff --git a/crates/libsy-llm-client/src/backend.rs b/crates/libsy-llm-client/src/backend.rs index fab446095..bb6a61552 100644 --- a/crates/libsy-llm-client/src/backend.rs +++ b/crates/libsy-llm-client/src/backend.rs @@ -175,6 +175,17 @@ impl Backend { self.config().forward_auth } + /// Whether a Responses backend can replay provider-encrypted reasoning. + /// + /// Hosted OpenAI-compatible providers are authenticated either with a + /// configured key or caller-forwarded credentials. Unauthenticated + /// Responses backends are typically local servers, which cannot consume + /// another provider's encrypted reasoning items. + pub(crate) fn supports_encrypted_reasoning(&self) -> bool { + matches!(self, Backend::OpenAiResponses(_)) + && (self.config().api_key.is_some() || self.config().forward_auth) + } + /// Applies only the caller credential accepted by this provider. pub(crate) fn apply_forwarded_auth( &self, diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rs index b5c139e21..1eb28beae 100644 --- a/crates/libsy-llm-client/src/client.rs +++ b/crates/libsy-llm-client/src/client.rs @@ -222,7 +222,7 @@ impl TranslatingLlmClient { strip_unsigned_thinking_blocks(&mut body); } if matches!(backend, Backend::OpenAiResponses(_)) { - strip_unsigned_responses_reasoning(&mut body); + normalize_responses_reasoning(&mut body, backend.supports_encrypted_reasoning()); } merge_extra_body(&mut body, backend.extra_body()); if matches!(backend, Backend::Anthropic(_)) { @@ -679,11 +679,12 @@ fn set_json_model(body: &mut Value, model: &str) { } } -// Removes plaintext reasoning items that strict Responses backends cannot replay. -// Signed reasoning keeps an empty content array: strict OpenAI requires that the -// array contain no plaintext, while OpenAI-compatible backends such as llama.cpp -// require the field itself to remain an array when replaying the item. -fn strip_unsigned_responses_reasoning(body: &mut Value) { +// Removes reasoning items that the selected Responses backend cannot replay. +// Strict authenticated providers retain signed encrypted reasoning with no +// plaintext. Unauthenticated local backends cannot consume another provider's +// encrypted reasoning, so they drop signed and unsigned reasoning alike while +// preserving messages and tool-call history. +fn normalize_responses_reasoning(body: &mut Value, preserve_signed: bool) { let Some(Value::Array(input)) = body.get_mut("input") else { return; }; @@ -698,10 +699,10 @@ fn strip_unsigned_responses_reasoning(body: &mut Value) { object.get("encrypted_content").and_then(Value::as_str), Some(encrypted_content) if !encrypted_content.is_empty() ); - if signed { + if signed && preserve_signed { object.insert("content".to_string(), Value::Array(Vec::new())); } - signed + signed && preserve_signed }); } @@ -908,6 +909,16 @@ mod tests { )] } + fn local_responses_map(base_url: &str) -> Vec { + let mut backend = config(base_url); + backend.api_key = None; + vec![ModelConfig::new( + "local", + Backend::OpenAiResponses(backend), + None, + )] + } + fn chat_map_with_retries(base_url: &str, max_retries: u32) -> Vec { vec![ModelConfig::new( "gpt", @@ -1482,6 +1493,72 @@ mod tests { Ok(()) } + // Encrypted hosted reasoning is opaque to a local Responses backend. Dropping + // it avoids llama.cpp rejecting a missing or empty `content` array while + // retaining the conversation and tool-call history it can consume. + #[tokio::test] + async fn local_responses_requests_drop_encrypted_reasoning_items() + -> std::result::Result<(), Box> { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/v1/responses")) + .and(|request: &wiremock::Request| { + let body: Value = serde_json::from_slice(&request.body).unwrap_or(Value::Null); + let Some(input) = body.get("input").and_then(Value::as_array) else { + return false; + }; + input + .iter() + .all(|item| item.get("type").and_then(Value::as_str) != Some("reasoning")) + && input.iter().any(|item| { + item.get("type").and_then(Value::as_str) == Some("function_call") + }) + && input.iter().any(|item| { + item.get("type").and_then(Value::as_str) == Some("function_call_output") + }) + }) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "id": "resp_local", + "object": "response", + "model": "local", + "status": "completed", + "output": [{ + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "ok"}] + }], + "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2} + }))) + .mount(&server) + .await; + + let client = + TranslatingLlmClient::new(&local_responses_map(&format!("{}/v1", server.uri())))?; + let raw = json!({ + "model": "client-facing", + "input": [ + {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "inspect"}]}, + { + "type": "reasoning", + "encrypted_content": "opaque-provider-reasoning" + }, + {"type": "function_call", "call_id": "call_1", "name": "shell", "arguments": "{}"}, + {"type": "function_call_output", "call_id": "call_1", "output": "ok"}, + {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "continue"}]} + ] + }); + + client + .call_rewrite_model_raw( + raw, + None, + Some(&ModelId::from("local")), + WireFormat::OpenAiResponses, + ) + .await?; + Ok(()) + } + // A router can serve earlier turns from an OpenAI target and later turns from // an Anthropic one, so the Anthropic leg must drop OpenAI-only fields the // caller keeps sending or the upstream rejects the whole request. From ab9b9e8fc5cdfe66aa0807bce8b7c4bf3f3b0a73 Mon Sep 17 00:00:00 2001 From: Sumanth Chandrupatla Date: Wed, 19 Aug 2026 07:33:12 -0500 Subject: [PATCH 4/5] refactor: configure Responses reasoning replay explicitly Signed-off-by: Sumanth Chandrupatla --- Cargo.lock | 1 + crates/libsy-llm-client/Cargo.toml | 1 + crates/libsy-llm-client/README.md | 7 +- crates/libsy-llm-client/src/backend.rs | 20 +-- crates/libsy-llm-client/src/client.rs | 34 +---- crates/libsy-llm-client/src/lib.rs | 2 + .../src/responses_reasoning.rs | 116 ++++++++++++++++++ crates/libsy-llm-client/src/run.rs | 1 + crates/switchyard-server/README.md | 4 + crates/switchyard-server/src/config.rs | 60 ++++++++- crates/switchyard-server/tests/server.rs | 1 + docs/getting_started.md | 3 + docs/reference/toml_schema.md | 18 +++ 13 files changed, 226 insertions(+), 42 deletions(-) create mode 100644 crates/libsy-llm-client/src/responses_reasoning.rs diff --git a/Cargo.lock b/Cargo.lock index 405dda9da..83cfc8ede 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2301,6 +2301,7 @@ dependencies = [ "opentelemetry_sdk", "parking_lot", "reqwest", + "serde", "serde_json", "switchyard-libsy", "switchyard-protocol", diff --git a/crates/libsy-llm-client/Cargo.toml b/crates/libsy-llm-client/Cargo.toml index 9a9ca6e0a..a347ed7cc 100644 --- a/crates/libsy-llm-client/Cargo.toml +++ b/crates/libsy-llm-client/Cargo.toml @@ -29,6 +29,7 @@ parking_lot.workspace = true http.workspace = true httpdate.workspace = true serde_json.workspace = true +serde.workspace = true tokio.workspace = true tracing.workspace = true tracing-opentelemetry.workspace = true diff --git a/crates/libsy-llm-client/README.md b/crates/libsy-llm-client/README.md index b62da05e1..0f90a0b8c 100644 --- a/crates/libsy-llm-client/README.md +++ b/crates/libsy-llm-client/README.md @@ -59,7 +59,8 @@ switchyard-translation = { path = "../switchyard-translation" } # for WireForm ```rust use std::collections::BTreeMap; use switchyard_llm_client::{ - Backend, HttpBackendConfig, ModelConfig, TranslatingLlmClient, + Backend, HttpBackendConfig, ModelConfig, ResponsesReasoningPolicy, + TranslatingLlmClient, }; fn build_client() -> switchyard_llm_client::Result { @@ -70,6 +71,7 @@ fn build_client() -> switchyard_llm_client::Result { extra_headers: BTreeMap::new(), extra_body: BTreeMap::new(), max_retries: 2, + responses_reasoning: ResponsesReasoningPolicy::default(), }; let models = [ModelConfig::new( @@ -246,6 +248,9 @@ fn build_multi_format_client( transport failures, timeouts, HTTP 408/429, and 5xx responses. Buffered body transport failures are retried; streaming body failures are not replayed after the response has been returned. +- `HttpBackendConfig::responses_reasoning` controls reasoning-item replay for + Responses backends. `PreserveEncrypted` keeps signed provider state without + plaintext; `Drop` removes reasoning while retaining messages and tool history. Retries replay the same upstream request to the same model. Each candidate's `max_retries` budget is exhausted before candidate fallback advances to the next diff --git a/crates/libsy-llm-client/src/backend.rs b/crates/libsy-llm-client/src/backend.rs index bb6a61552..47bef1ef0 100644 --- a/crates/libsy-llm-client/src/backend.rs +++ b/crates/libsy-llm-client/src/backend.rs @@ -11,6 +11,7 @@ use serde_json::Value; use switchyard_protocol::{Metadata, WireFormat}; use crate::error::{LlmClientError, Result, is_overflow_body}; +use crate::responses_reasoning::ResponsesReasoningPolicy; const ANTHROPIC_VERSION: &str = "2023-06-01"; @@ -56,6 +57,8 @@ pub struct HttpBackendConfig { pub extra_body: BTreeMap, /// Additional attempts after the initial upstream request. pub max_retries: u32, + /// Replay policy for OpenAI Responses reasoning items. + pub responses_reasoning: ResponsesReasoningPolicy, } impl fmt::Debug for HttpBackendConfig { @@ -67,6 +70,7 @@ impl fmt::Debug for HttpBackendConfig { .field("extra_header_names", &self.extra_headers.keys()) .field("extra_body_keys", &self.extra_body.keys()) .field("max_retries", &self.max_retries) + .field("responses_reasoning", &self.responses_reasoning) .finish() } } @@ -175,15 +179,12 @@ impl Backend { self.config().forward_auth } - /// Whether a Responses backend can replay provider-encrypted reasoning. - /// - /// Hosted OpenAI-compatible providers are authenticated either with a - /// configured key or caller-forwarded credentials. Unauthenticated - /// Responses backends are typically local servers, which cannot consume - /// another provider's encrypted reasoning items. - pub(crate) fn supports_encrypted_reasoning(&self) -> bool { - matches!(self, Backend::OpenAiResponses(_)) - && (self.config().api_key.is_some() || self.config().forward_auth) + /// The configured reasoning replay policy for a Responses backend. + pub(crate) fn responses_reasoning(&self) -> Option { + match self { + Backend::OpenAiResponses(config) => Some(config.responses_reasoning), + Backend::OpenAiChat(_) | Backend::Anthropic(_) => None, + } } /// Applies only the caller credential accepted by this provider. @@ -357,6 +358,7 @@ mod tests { extra_headers: BTreeMap::new(), extra_body: BTreeMap::new(), max_retries: 0, + responses_reasoning: ResponsesReasoningPolicy::default(), } } diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rs index 1eb28beae..93ad927a5 100644 --- a/crates/libsy-llm-client/src/client.rs +++ b/crates/libsy-llm-client/src/client.rs @@ -221,8 +221,8 @@ impl TranslatingLlmClient { strip_anthropic_incompatible_fields(&mut body); strip_unsigned_thinking_blocks(&mut body); } - if matches!(backend, Backend::OpenAiResponses(_)) { - normalize_responses_reasoning(&mut body, backend.supports_encrypted_reasoning()); + if let Some(policy) = backend.responses_reasoning() { + policy.normalize(&mut body); } merge_extra_body(&mut body, backend.extra_body()); if matches!(backend, Backend::Anthropic(_)) { @@ -679,33 +679,6 @@ fn set_json_model(body: &mut Value, model: &str) { } } -// Removes reasoning items that the selected Responses backend cannot replay. -// Strict authenticated providers retain signed encrypted reasoning with no -// plaintext. Unauthenticated local backends cannot consume another provider's -// encrypted reasoning, so they drop signed and unsigned reasoning alike while -// preserving messages and tool-call history. -fn normalize_responses_reasoning(body: &mut Value, preserve_signed: bool) { - let Some(Value::Array(input)) = body.get_mut("input") else { - return; - }; - input.retain_mut(|item| { - let Some(object) = item.as_object_mut() else { - return true; - }; - if object.get("type").and_then(Value::as_str) != Some("reasoning") { - return true; - } - let signed = matches!( - object.get("encrypted_content").and_then(Value::as_str), - Some(encrypted_content) if !encrypted_content.is_empty() - ); - if signed && preserve_signed { - object.insert("content".to_string(), Value::Array(Vec::new())); - } - signed && preserve_signed - }); -} - // Drops fields accepted by OpenAI-like APIs but rejected by Anthropic Messages. // // A router can serve earlier turns of a session from an OpenAI-format target and @@ -865,6 +838,7 @@ mod tests { extra_headers: BTreeMap::new(), extra_body: BTreeMap::new(), max_retries: 0, + responses_reasoning: crate::ResponsesReasoningPolicy::default(), } } @@ -911,7 +885,7 @@ mod tests { fn local_responses_map(base_url: &str) -> Vec { let mut backend = config(base_url); - backend.api_key = None; + backend.responses_reasoning = crate::ResponsesReasoningPolicy::Drop; vec![ModelConfig::new( "local", Backend::OpenAiResponses(backend), diff --git a/crates/libsy-llm-client/src/lib.rs b/crates/libsy-llm-client/src/lib.rs index 059675bbe..dd45d865c 100644 --- a/crates/libsy-llm-client/src/lib.rs +++ b/crates/libsy-llm-client/src/lib.rs @@ -23,6 +23,7 @@ pub mod metrics; mod observability; mod observation; pub mod raw; +mod responses_reasoning; pub mod run; pub use backend::{Backend, DEFAULT_MAX_RETRIES, HttpBackendConfig}; @@ -30,6 +31,7 @@ pub use client::{ModelConfig, TranslatingLlmClient}; pub use error::{LlmClientError, Result}; pub use observation::{LlmCallObservation, RunObservation, RunObserver}; pub use raw::RawResponse; +pub use responses_reasoning::ResponsesReasoningPolicy; pub use run::{ClientRouter, run}; pub use switchyard_translation::RawEventStream; diff --git a/crates/libsy-llm-client/src/responses_reasoning.rs b/crates/libsy-llm-client/src/responses_reasoning.rs new file mode 100644 index 000000000..fba6e868a --- /dev/null +++ b/crates/libsy-llm-client/src/responses_reasoning.rs @@ -0,0 +1,116 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Target-specific replay policy for OpenAI Responses reasoning items. + +use serde::Deserialize; +use serde_json::Value; + +/// Controls which Responses reasoning items are replayed to an upstream. +#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, PartialEq)] +#[serde(rename_all = "snake_case")] +pub enum ResponsesReasoningPolicy { + /// Preserve provider-encrypted reasoning but remove plaintext reasoning. + /// + /// This is the safe default for strict hosted Responses providers. + #[default] + PreserveEncrypted, + /// Drop all reasoning items while preserving messages and tool-call history. + /// + /// Use this for local Responses-compatible servers that cannot consume + /// another provider's encrypted reasoning representation. + Drop, +} + +impl ResponsesReasoningPolicy { + /// Normalizes a Responses request body for this replay policy. + pub(crate) fn normalize(self, body: &mut Value) { + let Some(Value::Array(input)) = body.get_mut("input") else { + return; + }; + input.retain_mut(|item| self.normalize_item(item)); + } + + fn normalize_item(self, item: &mut Value) -> bool { + let Some(object) = item.as_object_mut() else { + return true; + }; + if object.get("type").and_then(Value::as_str) != Some("reasoning") { + return true; + } + + let signed = matches!( + object.get("encrypted_content").and_then(Value::as_str), + Some(encrypted_content) if !encrypted_content.is_empty() + ); + if self == Self::PreserveEncrypted && signed { + object.insert("content".to_string(), Value::Array(Vec::new())); + true + } else { + false + } + } +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + fn mixed_history() -> Value { + json!({ + "input": [ + {"type": "message", "role": "user", "content": []}, + { + "type": "reasoning", + "content": [{"type": "reasoning_text", "text": "plaintext"}], + "encrypted_content": "" + }, + {"type": "function_call", "call_id": "call_1"}, + {"type": "function_call_output", "call_id": "call_1", "output": "ok"}, + { + "type": "reasoning", + "content": [{"type": "reasoning_text", "text": "must be removed"}], + "encrypted_content": "encrypted" + } + ] + }) + } + + #[test] + fn preserve_encrypted_drops_unsigned_and_clears_plaintext() { + let mut body = mixed_history(); + ResponsesReasoningPolicy::PreserveEncrypted.normalize(&mut body); + + let input = body["input"].as_array().expect("input array"); + let reasoning: Vec<&Value> = input + .iter() + .filter(|item| item["type"] == "reasoning") + .collect(); + assert_eq!(reasoning.len(), 1); + assert_eq!(reasoning[0]["encrypted_content"], "encrypted"); + assert_eq!(reasoning[0]["content"], json!([])); + assert!(input.iter().any(|item| item["type"] == "function_call")); + assert!( + input + .iter() + .any(|item| item["type"] == "function_call_output") + ); + } + + #[test] + fn drop_removes_all_reasoning_and_keeps_tool_history() { + let mut body = mixed_history(); + ResponsesReasoningPolicy::Drop.normalize(&mut body); + + let input = body["input"].as_array().expect("input array"); + assert!(input.iter().all(|item| item["type"] != "reasoning")); + assert!(input.iter().any(|item| item["type"] == "function_call")); + assert!( + input + .iter() + .any(|item| item["type"] == "function_call_output") + ); + } +} diff --git a/crates/libsy-llm-client/src/run.rs b/crates/libsy-llm-client/src/run.rs index 73d897c39..8eba79768 100644 --- a/crates/libsy-llm-client/src/run.rs +++ b/crates/libsy-llm-client/src/run.rs @@ -685,6 +685,7 @@ mod tests { extra_headers: BTreeMap::new(), extra_body: BTreeMap::new(), max_retries: 2, + responses_reasoning: crate::ResponsesReasoningPolicy::default(), }) }; let client = Arc::new( diff --git a/crates/switchyard-server/README.md b/crates/switchyard-server/README.md index cf107fecc..3086feed2 100644 --- a/crates/switchyard-server/README.md +++ b/crates/switchyard-server/README.md @@ -85,6 +85,10 @@ Target-level `extra_body` values are shallow-merged into the upstream request wh the request does not already contain that key. `max_retries` defaults to `2` and applies to transport failures, timeouts, HTTP 408/429, and 5xx responses. +For an `openai_responses` client, `responses_reasoning` defaults to +`preserve_encrypted`. Set it to `drop` for a local compatible server that cannot +replay another provider's encrypted reasoning items; messages and tool history +are retained. Random-route `weights` are relative, follow target order, and do not need to sum to one. Omit them for equal weighting. The optional `seed` reproduces the selection sequence for the same call order. diff --git a/crates/switchyard-server/src/config.rs b/crates/switchyard-server/src/config.rs index a78695cd5..2882589f7 100644 --- a/crates/switchyard-server/src/config.rs +++ b/crates/switchyard-server/src/config.rs @@ -18,7 +18,7 @@ use serde::Deserialize; use serde_json::Value; use switchyard_llm_client::{ Backend, ClientRouter, DEFAULT_MAX_RETRIES, HttpBackendConfig, ModelConfig, - TranslatingLlmClient, + ResponsesReasoningPolicy, TranslatingLlmClient, }; use switchyard_protocol::{ModelId, RoutedLlmClient}; @@ -256,6 +256,7 @@ struct LlmClientConfig { extra_headers: BTreeMap, #[serde(default = "default_max_retries")] max_retries: u32, + responses_reasoning: Option, } #[derive(Debug, Deserialize)] @@ -267,7 +268,7 @@ struct TargetConfig { extra_body: BTreeMap, } -#[derive(Clone, Copy, Debug, Deserialize)] +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq)] enum ClientFormat { #[serde(rename = "openai_chat")] OpenAiChat, @@ -875,6 +876,11 @@ fn build_backend( "llm client {client_name} cannot set both forward_auth and api_key_env" ))); } + if config.responses_reasoning.is_some() && config.format != ClientFormat::OpenAiResponses { + return Err(ServerError::new(format!( + "llm client {client_name} responses_reasoning is only valid for openai_responses" + ))); + } let api_key = config .api_key_env .as_deref() @@ -904,6 +910,7 @@ fn build_backend( extra_headers: config.extra_headers.clone(), extra_body: extra_body.clone(), max_retries: config.max_retries, + responses_reasoning: config.responses_reasoning.unwrap_or_default(), }; Ok(match config.format { ClientFormat::OpenAiChat => Backend::OpenAiChat(http), @@ -1655,6 +1662,55 @@ target = "azure" Ok(()) } + #[test] + fn responses_reasoning_defaults_and_accepts_drop() -> ServerResult<()> { + let default: ServerConfig = toml::from_str(VALID_CONFIG) + .map_err(|error| ServerError::new(format!("failed to parse config: {error}")))?; + let responses = default + .llm_clients + .get("responses") + .ok_or_else(|| ServerError::new("responses llm client is missing"))?; + let backend = build_backend("responses", responses, &BTreeMap::new())?; + let Backend::OpenAiResponses(http) = backend else { + return Err(ServerError::new("expected Responses backend")); + }; + assert_eq!( + http.responses_reasoning, + ResponsesReasoningPolicy::PreserveEncrypted + ); + + let configured = VALID_CONFIG.replacen( + "[llm_clients.responses]\nformat = \"openai_responses\"\nbase_url = \"https://example.test/v1\"", + "[llm_clients.responses]\nformat = \"openai_responses\"\nbase_url = \"https://example.test/v1\"\nresponses_reasoning = \"drop\"", + 1, + ); + let config: ServerConfig = toml::from_str(&configured) + .map_err(|error| ServerError::new(format!("failed to parse config: {error}")))?; + let responses = config + .llm_clients + .get("responses") + .ok_or_else(|| ServerError::new("responses llm client is missing"))?; + let backend = build_backend("responses", responses, &BTreeMap::new())?; + let Backend::OpenAiResponses(http) = backend else { + return Err(ServerError::new("expected Responses backend")); + }; + assert_eq!(http.responses_reasoning, ResponsesReasoningPolicy::Drop); + Ok(()) + } + + #[test] + fn responses_reasoning_rejects_other_formats() { + let invalid = VALID_CONFIG.replacen( + "format = \"openai_chat\"", + "format = \"openai_chat\"\nresponses_reasoning = \"drop\"", + 1, + ); + assert!( + error_message(&invalid) + .contains("responses_reasoning is only valid for openai_responses") + ); + } + #[test] fn rejects_headers_that_switchyard_sets() { let cases = [ diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs index fc15ba3f8..83e97a7c6 100644 --- a/crates/switchyard-server/tests/server.rs +++ b/crates/switchyard-server/tests/server.rs @@ -372,6 +372,7 @@ fn random_state(base_url: &str, routes: &[(&str, &[&str])]) -> TestResult]` | Key | Required | Default | Meaning | From 0971862f062930d12da2a7a87b7d187d1482d0f0 Mon Sep 17 00:00:00 2001 From: Sumanth Chandrupatla Date: Wed, 19 Aug 2026 07:38:25 -0500 Subject: [PATCH 5/5] refactor: preserve backend configuration compatibility Signed-off-by: Sumanth Chandrupatla --- crates/libsy-llm-client/README.md | 8 +++--- crates/libsy-llm-client/src/backend.rs | 13 ---------- crates/libsy-llm-client/src/client.rs | 29 ++++++++++++++------- crates/libsy-llm-client/src/run.rs | 1 - crates/switchyard-server/src/config.rs | 33 ++++++++++-------------- crates/switchyard-server/tests/server.rs | 1 - 6 files changed, 36 insertions(+), 49 deletions(-) diff --git a/crates/libsy-llm-client/README.md b/crates/libsy-llm-client/README.md index 0f90a0b8c..d5e7f1175 100644 --- a/crates/libsy-llm-client/README.md +++ b/crates/libsy-llm-client/README.md @@ -59,8 +59,7 @@ switchyard-translation = { path = "../switchyard-translation" } # for WireForm ```rust use std::collections::BTreeMap; use switchyard_llm_client::{ - Backend, HttpBackendConfig, ModelConfig, ResponsesReasoningPolicy, - TranslatingLlmClient, + Backend, HttpBackendConfig, ModelConfig, TranslatingLlmClient, }; fn build_client() -> switchyard_llm_client::Result { @@ -71,7 +70,6 @@ fn build_client() -> switchyard_llm_client::Result { extra_headers: BTreeMap::new(), extra_body: BTreeMap::new(), max_retries: 2, - responses_reasoning: ResponsesReasoningPolicy::default(), }; let models = [ModelConfig::new( @@ -248,8 +246,8 @@ fn build_multi_format_client( transport failures, timeouts, HTTP 408/429, and 5xx responses. Buffered body transport failures are retried; streaming body failures are not replayed after the response has been returned. -- `HttpBackendConfig::responses_reasoning` controls reasoning-item replay for - Responses backends. `PreserveEncrypted` keeps signed provider state without +- `ModelConfig::with_responses_reasoning` controls reasoning-item replay for a + Responses model. `PreserveEncrypted` keeps signed provider state without plaintext; `Drop` removes reasoning while retaining messages and tool history. Retries replay the same upstream request to the same model. Each candidate's diff --git a/crates/libsy-llm-client/src/backend.rs b/crates/libsy-llm-client/src/backend.rs index 47bef1ef0..fab446095 100644 --- a/crates/libsy-llm-client/src/backend.rs +++ b/crates/libsy-llm-client/src/backend.rs @@ -11,7 +11,6 @@ use serde_json::Value; use switchyard_protocol::{Metadata, WireFormat}; use crate::error::{LlmClientError, Result, is_overflow_body}; -use crate::responses_reasoning::ResponsesReasoningPolicy; const ANTHROPIC_VERSION: &str = "2023-06-01"; @@ -57,8 +56,6 @@ pub struct HttpBackendConfig { pub extra_body: BTreeMap, /// Additional attempts after the initial upstream request. pub max_retries: u32, - /// Replay policy for OpenAI Responses reasoning items. - pub responses_reasoning: ResponsesReasoningPolicy, } impl fmt::Debug for HttpBackendConfig { @@ -70,7 +67,6 @@ impl fmt::Debug for HttpBackendConfig { .field("extra_header_names", &self.extra_headers.keys()) .field("extra_body_keys", &self.extra_body.keys()) .field("max_retries", &self.max_retries) - .field("responses_reasoning", &self.responses_reasoning) .finish() } } @@ -179,14 +175,6 @@ impl Backend { self.config().forward_auth } - /// The configured reasoning replay policy for a Responses backend. - pub(crate) fn responses_reasoning(&self) -> Option { - match self { - Backend::OpenAiResponses(config) => Some(config.responses_reasoning), - Backend::OpenAiChat(_) | Backend::Anthropic(_) => None, - } - } - /// Applies only the caller credential accepted by this provider. pub(crate) fn apply_forwarded_auth( &self, @@ -358,7 +346,6 @@ mod tests { extra_headers: BTreeMap::new(), extra_body: BTreeMap::new(), max_retries: 0, - responses_reasoning: ResponsesReasoningPolicy::default(), } } diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rs index 93ad927a5..616871801 100644 --- a/crates/libsy-llm-client/src/client.rs +++ b/crates/libsy-llm-client/src/client.rs @@ -62,6 +62,7 @@ pub struct ModelConfig { model_name: ModelId, default_backend: Backend, other_backends: Option>, + responses_reasoning: crate::ResponsesReasoningPolicy, } impl ModelConfig { @@ -76,8 +77,16 @@ impl ModelConfig { model_name: model_name.into(), default_backend, other_backends, + responses_reasoning: crate::ResponsesReasoningPolicy::default(), } } + + /// Sets how Responses reasoning items are replayed to this model. + #[must_use] + pub fn with_responses_reasoning(mut self, policy: crate::ResponsesReasoningPolicy) -> Self { + self.responses_reasoning = policy; + self + } } /// A client that dispatches neutral-IR requests to per-model HTTP backends. @@ -221,8 +230,12 @@ impl TranslatingLlmClient { strip_anthropic_incompatible_fields(&mut body); strip_unsigned_thinking_blocks(&mut body); } - if let Some(policy) = backend.responses_reasoning() { - policy.normalize(&mut body); + if matches!(backend, Backend::OpenAiResponses(_)) { + self.model_to_config + .get(model) + .map(|config| config.responses_reasoning) + .unwrap_or_default() + .normalize(&mut body); } merge_extra_body(&mut body, backend.extra_body()); if matches!(backend, Backend::Anthropic(_)) { @@ -838,7 +851,6 @@ mod tests { extra_headers: BTreeMap::new(), extra_body: BTreeMap::new(), max_retries: 0, - responses_reasoning: crate::ResponsesReasoningPolicy::default(), } } @@ -884,13 +896,10 @@ mod tests { } fn local_responses_map(base_url: &str) -> Vec { - let mut backend = config(base_url); - backend.responses_reasoning = crate::ResponsesReasoningPolicy::Drop; - vec![ModelConfig::new( - "local", - Backend::OpenAiResponses(backend), - None, - )] + vec![ + ModelConfig::new("local", Backend::OpenAiResponses(config(base_url)), None) + .with_responses_reasoning(crate::ResponsesReasoningPolicy::Drop), + ] } fn chat_map_with_retries(base_url: &str, max_retries: u32) -> Vec { diff --git a/crates/libsy-llm-client/src/run.rs b/crates/libsy-llm-client/src/run.rs index 8eba79768..73d897c39 100644 --- a/crates/libsy-llm-client/src/run.rs +++ b/crates/libsy-llm-client/src/run.rs @@ -685,7 +685,6 @@ mod tests { extra_headers: BTreeMap::new(), extra_body: BTreeMap::new(), max_retries: 2, - responses_reasoning: crate::ResponsesReasoningPolicy::default(), }) }; let client = Arc::new( diff --git a/crates/switchyard-server/src/config.rs b/crates/switchyard-server/src/config.rs index 2882589f7..71f3129c9 100644 --- a/crates/switchyard-server/src/config.rs +++ b/crates/switchyard-server/src/config.rs @@ -135,11 +135,14 @@ impl ServerConfig { let model_configs = models_by_client .get_mut(&target.llm_client) .ok_or_else(|| ServerError::new("validated llm client was not initialized"))?; - model_configs.push(ModelConfig::new( - target.id.clone(), - build_backend(&target.llm_client, client_config, &target.extra_body)?, - None, - )); + model_configs.push( + ModelConfig::new( + target.id.clone(), + build_backend(&target.llm_client, client_config, &target.extra_body)?, + None, + ) + .with_responses_reasoning(client_config.responses_reasoning.unwrap_or_default()), + ); } let mut clients = BTreeMap::new(); @@ -910,7 +913,6 @@ fn build_backend( extra_headers: config.extra_headers.clone(), extra_body: extra_body.clone(), max_retries: config.max_retries, - responses_reasoning: config.responses_reasoning.unwrap_or_default(), }; Ok(match config.format { ClientFormat::OpenAiChat => Backend::OpenAiChat(http), @@ -1670,14 +1672,7 @@ target = "azure" .llm_clients .get("responses") .ok_or_else(|| ServerError::new("responses llm client is missing"))?; - let backend = build_backend("responses", responses, &BTreeMap::new())?; - let Backend::OpenAiResponses(http) = backend else { - return Err(ServerError::new("expected Responses backend")); - }; - assert_eq!( - http.responses_reasoning, - ResponsesReasoningPolicy::PreserveEncrypted - ); + assert_eq!(responses.responses_reasoning, None); let configured = VALID_CONFIG.replacen( "[llm_clients.responses]\nformat = \"openai_responses\"\nbase_url = \"https://example.test/v1\"", @@ -1690,11 +1685,11 @@ target = "azure" .llm_clients .get("responses") .ok_or_else(|| ServerError::new("responses llm client is missing"))?; - let backend = build_backend("responses", responses, &BTreeMap::new())?; - let Backend::OpenAiResponses(http) = backend else { - return Err(ServerError::new("expected Responses backend")); - }; - assert_eq!(http.responses_reasoning, ResponsesReasoningPolicy::Drop); + assert_eq!( + responses.responses_reasoning, + Some(ResponsesReasoningPolicy::Drop) + ); + server_state_from_toml(&configured)?; Ok(()) } diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs index 83e97a7c6..fc15ba3f8 100644 --- a/crates/switchyard-server/tests/server.rs +++ b/crates/switchyard-server/tests/server.rs @@ -372,7 +372,6 @@ fn random_state(base_url: &str, routes: &[(&str, &[&str])]) -> TestResult