From dd0fc93180f04feb5bd2e6ef08607f8e4deb9a4f Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sun, 20 Sep 2026 11:00:04 +0300 Subject: [PATCH 1/7] chore(deps): update submodule pointers for vendor and wiki Updated the pinned commits for the tinyinference, tinytools, and wiki submodules to incorporate upstream changes. Auto-committed-on: dragonfly Co-authored-by: Medulla --- vendor/tinyinference | 2 +- vendor/tinytools | 2 +- wiki | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/vendor/tinyinference b/vendor/tinyinference index 2c3ff818..aeaecda2 160000 --- a/vendor/tinyinference +++ b/vendor/tinyinference @@ -1 +1 @@ -Subproject commit 2c3ff818f55f0644970eeaeee51b60cb9fa10100 +Subproject commit aeaecda26677161713b3a993872813e2a3594c21 diff --git a/vendor/tinytools b/vendor/tinytools index e347becd..9ae1d44d 160000 --- a/vendor/tinytools +++ b/vendor/tinytools @@ -1 +1 @@ -Subproject commit e347becd7b83bd9a07e65b69b4294c485fc86e26 +Subproject commit 9ae1d44de00863bfff5e3549063d14d57db15c21 diff --git a/wiki b/wiki index 6c8cf71e..5b8f5927 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit 6c8cf71e1312d350439dc243ac6e09273ec90d25 +Subproject commit 5b8f5927caa2ba0730985e686bf5ef5550e53e6d From b518ea77fb36957573d8c04723252345a4a93aeb Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sun, 20 Sep 2026 11:23:56 +0300 Subject: [PATCH 2/7] fix: support updated upstream submodule APIs Co-authored-by: Medulla --- .../src/orchestration/test.rs | 2 +- crates/tinyagents-graph/src/todos/test.rs | 4 ++- .../src/agent_loop/model_call.rs | 1 + .../tinyagents-harness/src/agent_loop/test.rs | 12 ++++++++ .../src/agent_loop/tools.rs | 30 +++++++++++++++++++ .../tinyagents-harness/src/context/stats.rs | 3 ++ crates/tinyagents-harness/src/context/test.rs | 9 ++++++ crates/tinyagents-harness/src/error.rs | 1 + .../src/middleware/library/observe.rs | 20 +++++++++++++ .../tinyagents-harness/src/middleware/test.rs | 4 +++ .../src/providers/claude_agent_sdk/mod.rs | 3 +- .../src/providers/claude_code/mod.rs | 15 +++++++++- .../src/providers/claude_code/mod_tests.rs | 18 +++++++++++ .../tinyagents-harness/src/steering/test.rs | 4 +++ .../src/summarization/render.rs | 5 ++++ .../src/summarization/test.rs | 10 +++++++ .../src/summarization/types.rs | 3 ++ .../src/token_estimation.rs | 1 + .../examples/agent_loop_tools.rs | 4 +++ .../examples/basic_graph.rs | 1 + .../examples/local_model_probe.rs | 2 ++ .../tests/context_and_schema_compaction.rs | 6 ++++ .../tests/dependency_boundary.rs | 2 +- .../tests/e2e_agent_graph.rs | 4 +++ .../tests/e2e_budget.rs | 8 +++++ .../tests/e2e_control_and_steer.rs | 4 +-- .../tests/e2e_fuzz_graph_agents.rs | 4 +++ .../tests/e2e_graph_task_dispatch.rs | 4 +++ .../tests/e2e_graph_todos.rs | 4 +++ .../tests/e2e_harness_mock_capabilities.rs | 4 +++ .../tests/e2e_harness_provider_contracts.rs | 6 ++++ .../tests/e2e_middleware.rs | 4 +++ .../tests/e2e_middleware_parser_contracts.rs | 1 + .../tests/e2e_observability.rs | 4 +++ .../tests/e2e_prompt_cache_kv.rs | 2 ++ .../tests/e2e_reasoning_and_selection.rs | 2 ++ .../tests/e2e_subagents.rs | 4 +++ .../tests/e2e_unknown_tool_policy.rs | 4 +++ .../tests/feature_harness_agent_loop.rs | 4 +++ .../tests/feature_harness_structured.rs | 6 ++++ .../tests/feature_infra_resilience.rs | 4 +++ .../tests/harness_agent_loop.rs | 4 +++ .../tests/tool_deferral.rs | 4 +++ .../tests/wave2_cache_retry_after.rs | 2 ++ 44 files changed, 236 insertions(+), 7 deletions(-) diff --git a/crates/tinyagents-graph/src/orchestration/test.rs b/crates/tinyagents-graph/src/orchestration/test.rs index 164824bf..7921ec15 100644 --- a/crates/tinyagents-graph/src/orchestration/test.rs +++ b/crates/tinyagents-graph/src/orchestration/test.rs @@ -40,7 +40,7 @@ fn raw(result: &ToolResult) -> &serde_json::Value { .iter() .find_map(|content| match content { ToolContent::Json { data } => Some(data), - ToolContent::Text { .. } => None, + ToolContent::Text { .. } | ToolContent::Image { .. } | ToolContent::File { .. } => None, }) .expect("orchestration tool returns a JSON payload") } diff --git a/crates/tinyagents-graph/src/todos/test.rs b/crates/tinyagents-graph/src/todos/test.rs index 6056a508..b4c5b6d0 100644 --- a/crates/tinyagents-graph/src/todos/test.rs +++ b/crates/tinyagents-graph/src/todos/test.rs @@ -526,7 +526,9 @@ mod tool_tests { .iter() .find_map(|block| match block { ToolContent::Json { data } => Some(data), - ToolContent::Text { .. } => None, + ToolContent::Text { .. } | ToolContent::Image { .. } | ToolContent::File { .. } => { + None + } }) .expect("successful todo result has a JSON payload") } diff --git a/crates/tinyagents-harness/src/agent_loop/model_call.rs b/crates/tinyagents-harness/src/agent_loop/model_call.rs index 0070d30a..e8923839 100644 --- a/crates/tinyagents-harness/src/agent_loop/model_call.rs +++ b/crates/tinyagents-harness/src/agent_loop/model_call.rs @@ -395,6 +395,7 @@ impl AgentHarness { call_id: call.id.clone(), content: serde_json::to_string(&call.arguments).unwrap_or_default(), tool_name: Some(call.name.clone()), + content_index: None, }), }); } diff --git a/crates/tinyagents-harness/src/agent_loop/test.rs b/crates/tinyagents-harness/src/agent_loop/test.rs index 32dbc104..e8a55fcc 100644 --- a/crates/tinyagents-harness/src/agent_loop/test.rs +++ b/crates/tinyagents-harness/src/agent_loop/test.rs @@ -306,6 +306,8 @@ fn tool_call_response(id: &str, name: &str, arguments: serde_json::Value) -> Mod content: Vec::new(), tool_calls: vec![ToolCall::new(id, name, arguments)], usage: Some(Usage::new(7, 3)), + + origin: None, }, usage: Some(Usage::new(7, 3)), finish_reason: Some("tool_calls".to_string()), @@ -333,6 +335,8 @@ fn invalid_tool_call_response(id: &str, name: &str, raw: &str) -> ModelResponse content: Vec::new(), tool_calls: vec![ToolCall::invalid(id, name, raw, reason)], usage: Some(Usage::new(7, 3)), + + origin: None, }, usage: Some(Usage::new(7, 3)), finish_reason: Some("tool_calls".to_string()), @@ -353,6 +357,8 @@ fn text_response(text: &str, input: u64, output: u64) -> ModelResponse { content: vec![ContentBlock::Text(text.to_string())], tool_calls: Vec::new(), usage: Some(Usage::new(input, output)), + + origin: None, }, usage: Some(Usage::new(input, output)), finish_reason: Some("stop".to_string()), @@ -377,6 +383,8 @@ fn truncated_empty_response(reasoning_tokens: u64) -> ModelResponse { content: Vec::new(), tool_calls: Vec::new(), usage: Some(Usage::new(4, reasoning_tokens)), + + origin: None, }, usage: Some(Usage::new(4, reasoning_tokens)), finish_reason: Some("length".to_string()), @@ -2919,6 +2927,7 @@ async fn streaming_middleware_can_suppress_a_standalone_tool_delta() { call_id: "blocked-call".to_string(), content: "{}".to_string(), tool_name: Some("blocked".to_string()), + content_index: None, }), ModelStreamItem::Completed(terminal), ])), @@ -2964,6 +2973,7 @@ async fn streaming_tool_delta_transform_controls_terminal_dispatch() { call_id: "raw-call".to_string(), content: r#"{"raw":true}"#.to_string(), tool_name: Some("blocked".to_string()), + content_index: None, }), ModelStreamItem::Completed(terminal), ])), @@ -4088,6 +4098,8 @@ fn multi_tool_call_response(calls: Vec<(&str, &str)>) -> ModelResponse { content: Vec::new(), tool_calls, usage: Some(Usage::new(7, 3)), + + origin: None, }, usage: Some(Usage::new(7, 3)), finish_reason: Some("tool_calls".to_string()), diff --git a/crates/tinyagents-harness/src/agent_loop/tools.rs b/crates/tinyagents-harness/src/agent_loop/tools.rs index 984e5084..804ece4b 100644 --- a/crates/tinyagents-harness/src/agent_loop/tools.rs +++ b/crates/tinyagents-harness/src/agent_loop/tools.rs @@ -1229,6 +1229,35 @@ fn tool_message_from_result( .map(|block| match block { tinytools::ToolContent::Text { text } => ContentBlock::Text(text.clone()), tinytools::ToolContent::Json { data } => ContentBlock::Json(data.clone()), + tinytools::ToolContent::Image { media_type, data } => { + let url = match data { + tinytools::ImageData::Base64(data) => { + format!("data:{media_type};base64,{data}") + } + tinytools::ImageData::Url(url) => url.clone(), + }; + ContentBlock::Image(tinyinference_llm::message::ImageRef { + url, + mime_type: Some(media_type.clone()), + }) + } + tinytools::ToolContent::File { + media_type, data, .. + } => { + use tinyinference_llm::message::MediaRef; + let media = match data { + tinytools::FileData::Base64(data) => MediaRef::Base64 { + data: data.clone(), + media_type: media_type.clone(), + }, + tinytools::FileData::Url(url) => MediaRef::url(url.clone()), + tinytools::FileData::Path(path) => MediaRef::Path { + path: path.clone(), + media_type: Some(media_type.clone()), + }, + }; + ContentBlock::Document(media) + } }) .collect() }; @@ -1428,6 +1457,7 @@ mod canonical_result_tests { ], is_error: true, markdown_formatted: Some("## compact failure".to_string()), + ..ToolResult::default() } } diff --git a/crates/tinyagents-harness/src/context/stats.rs b/crates/tinyagents-harness/src/context/stats.rs index 67bcce0e..511fe662 100644 --- a/crates/tinyagents-harness/src/context/stats.rs +++ b/crates/tinyagents-harness/src/context/stats.rs @@ -51,6 +51,7 @@ pub fn context_statistics(messages: &[Message]) -> ContextStatistics { } &message.content } + Message::Custom(_) => continue, }; for block in content { match block { @@ -61,6 +62,7 @@ pub fn context_statistics(messages: &[Message]) -> ContextStatistics { stats.text_chars += value.to_string().chars().count(); } ContentBlock::Image(_) => stats.images += 1, + ContentBlock::Audio(_) | ContentBlock::Video(_) | ContentBlock::Document(_) => {} ContentBlock::RedactedThinking { .. } => {} } } @@ -82,6 +84,7 @@ pub fn estimate_context_tokens(messages: &[Message], tokenize: impl Fn(&str) -> Message::User(message) => &message.content, Message::Assistant(message) => &message.content, Message::Tool(message) => &message.content, + Message::Custom(_) => return 0, }; let mut visible = content .iter() diff --git a/crates/tinyagents-harness/src/context/test.rs b/crates/tinyagents-harness/src/context/test.rs index e4dfea70..6c5be274 100644 --- a/crates/tinyagents-harness/src/context/test.rs +++ b/crates/tinyagents-harness/src/context/test.rs @@ -373,6 +373,8 @@ fn context_statistics_preserve_tool_request_result_pairing_and_image_counts() { content: vec![ContentBlock::Text("call it".into())], tool_calls: vec![ToolCall::new("call-1", "lookup", serde_json::json!({}))], usage: None, + + origin: None, }), Message::Tool(tinyinference_llm::message::ToolMessage { tool_call_id: "call-1".into(), @@ -435,6 +437,9 @@ fn token_estimation_includes_structured_blocks_for_every_role() { let messages = vec![ Message::System(SystemMessage { content: vec![ContentBlock::ProviderExtension(json.clone())], + sections: Default::default(), + tools_added: Vec::new(), + tools_removed: Vec::new(), }), Message::User(UserMessage { content: vec![ContentBlock::Json(json.clone())], @@ -444,6 +449,8 @@ fn token_estimation_includes_structured_blocks_for_every_role() { content: vec![ContentBlock::ProviderExtension(json.clone())], tool_calls: vec![], usage: None, + + origin: None, }), Message::Tool(ToolMessage { tool_call_id: "call".into(), @@ -472,6 +479,8 @@ fn token_estimation_includes_assistant_tool_names_and_arguments() { serde_json::json!({"query": "one two three"}), )], usage: None, + + origin: None, })]; let rendered = std::cell::RefCell::new(String::new()); diff --git a/crates/tinyagents-harness/src/error.rs b/crates/tinyagents-harness/src/error.rs index d8288464..ddc6b9f9 100644 --- a/crates/tinyagents-harness/src/error.rs +++ b/crates/tinyagents-harness/src/error.rs @@ -257,6 +257,7 @@ impl From for TinyAgentsError { tinyinference_llm::Error::Validation(message) => Self::Validation(message), tinyinference_llm::Error::Serialization(error) => Self::Serialization(error), tinyinference_llm::Error::Catalog(message) => Self::Model(message), + tinyinference_llm::Error::Unsupported(message) => Self::Model(message), } } } diff --git a/crates/tinyagents-harness/src/middleware/library/observe.rs b/crates/tinyagents-harness/src/middleware/library/observe.rs index 52ed83ab..2192cf24 100644 --- a/crates/tinyagents-harness/src/middleware/library/observe.rs +++ b/crates/tinyagents-harness/src/middleware/library/observe.rs @@ -265,6 +265,26 @@ impl Middleware for RedactionM } } ToolContent::Json { data } => hits += self.redact_value(data), + ToolContent::Image { data, .. } => match data { + tinytools::ImageData::Base64(value) | tinytools::ImageData::Url(value) => { + let (redacted, count) = self.redact(value); + if count > 0 { + *value = redacted; + hits += count; + } + } + }, + ToolContent::File { data, .. } => match data { + tinytools::FileData::Base64(value) + | tinytools::FileData::Url(value) + | tinytools::FileData::Path(value) => { + let (redacted, count) = self.redact(value); + if count > 0 { + *value = redacted; + hits += count; + } + } + }, } } if let Some(markdown) = &mut result.markdown_formatted { diff --git a/crates/tinyagents-harness/src/middleware/test.rs b/crates/tinyagents-harness/src/middleware/test.rs index ad48575c..625a2925 100644 --- a/crates/tinyagents-harness/src/middleware/test.rs +++ b/crates/tinyagents-harness/src/middleware/test.rs @@ -34,6 +34,8 @@ fn response_with_usage(usage: Usage) -> ModelResponse { content: vec![ContentBlock::Text("ok".to_string())], tool_calls: Vec::new(), usage: None, + + origin: None, }, usage: Some(usage), finish_reason: None, @@ -1073,6 +1075,8 @@ fn response_text(text: &str) -> ModelResponse { content: vec![ContentBlock::Text(text.to_string())], tool_calls: Vec::new(), usage: None, + + origin: None, }, usage: None, finish_reason: None, diff --git a/crates/tinyagents-harness/src/providers/claude_agent_sdk/mod.rs b/crates/tinyagents-harness/src/providers/claude_agent_sdk/mod.rs index 914eaf3e..23e1d4ef 100644 --- a/crates/tinyagents-harness/src/providers/claude_agent_sdk/mod.rs +++ b/crates/tinyagents-harness/src/providers/claude_agent_sdk/mod.rs @@ -107,7 +107,7 @@ fn build_invocation( fn render_transcript(messages: &[Message]) -> String { let non_system: Vec<&Message> = messages .iter() - .filter(|message| !matches!(message, Message::System(_))) + .filter(|message| !matches!(message, Message::System(_) | Message::Custom(_))) .collect(); if non_system.len() == 1 { return non_system[0].text(); @@ -121,6 +121,7 @@ fn render_transcript(messages: &[Message]) -> String { Message::Assistant(_) => "ASSISTANT", Message::Tool(_) => "TOOL", Message::System(_) => unreachable!("system messages were filtered"), + Message::Custom(_) => unreachable!("custom messages were filtered"), }; format!("[{role}]\n{}\n[/{role}]", message.text()) }) diff --git a/crates/tinyagents-harness/src/providers/claude_code/mod.rs b/crates/tinyagents-harness/src/providers/claude_code/mod.rs index 82e4a992..8095576f 100644 --- a/crates/tinyagents-harness/src/providers/claude_code/mod.rs +++ b/crates/tinyagents-harness/src/providers/claude_code/mod.rs @@ -323,7 +323,13 @@ fn thread_key_from_request(request: &ModelRequest) -> String { /// first, then any structured [`ResponseFormat`] is appended as a trailing /// system instruction (see [`response_format_instruction`]). fn request_messages(request: &ModelRequest) -> Vec { - let mut messages = coalesce_tool_results(&request.messages); + let provider_messages: Vec<_> = request + .messages + .iter() + .filter(|message| !matches!(message, Message::Custom(_))) + .cloned() + .collect(); + let mut messages = coalesce_tool_results(&provider_messages); if !request.tools.is_empty() { messages = with_tool_instructions(&messages, &request.tools, &request.tool_choice); } @@ -338,12 +344,14 @@ fn request_messages(request: &ModelRequest) -> Vec { Message::User(_) => "user", Message::Assistant(_) => "assistant", Message::Tool(_) => "tool", + Message::Custom(_) => unreachable!("custom messages were filtered"), }; let content = match message { Message::System(value) => render_content(&value.content), Message::User(value) => render_content(&value.content), Message::Assistant(value) => render_content(&value.content), Message::Tool(value) => render_content(&value.content), + Message::Custom(_) => unreachable!("custom messages were filtered"), }; ChatMessage::new(role, content) }) @@ -387,6 +395,9 @@ fn render_content(content: &[ContentBlock]) -> String { } ContentBlock::Thinking { text, .. } => Some(text.clone()), ContentBlock::RedactedThinking { .. } => None, + ContentBlock::Audio(_) => Some("[audio content]".to_string()), + ContentBlock::Video(_) => Some("[video content]".to_string()), + ContentBlock::Document(_) => Some("[document content]".to_string()), }) .collect::>() // Content-block boundaries carry no implicit whitespace. Inserting a @@ -416,6 +427,8 @@ fn model_response(response: ChatResponse) -> ModelResponse { content: response.text.into_iter().map(ContentBlock::Text).collect(), tool_calls: Vec::new(), usage, + + origin: None, }, usage, finish_reason: Some("stop".into()), diff --git a/crates/tinyagents-harness/src/providers/claude_code/mod_tests.rs b/crates/tinyagents-harness/src/providers/claude_code/mod_tests.rs index fe74a06d..472a0068 100644 --- a/crates/tinyagents-harness/src/providers/claude_code/mod_tests.rs +++ b/crates/tinyagents-harness/src/providers/claude_code/mod_tests.rs @@ -236,3 +236,21 @@ fn request_rendering_keeps_private_image_marker_text_literal() { assert_eq!(content[0]["text"], "literal "); assert_eq!(content[1]["text"], "[OH_IMAGE:data:image/png;base64,QUJD]"); } + +#[test] +fn request_messages_filter_host_custom_records() { + let request = ModelRequest::new(vec![ + Message::user("hello"), + Message::Custom(tinyinference_llm::message::CustomMessage { + kind: "compaction".into(), + payload: serde_json::json!({"summary": "host-only"}), + display: Some("host-only".into()), + }), + ]); + + let messages = request_messages(&request); + + assert_eq!(messages.len(), 1); + assert_eq!(messages[0].role, "user"); + assert_eq!(messages[0].content, "hello"); +} diff --git a/crates/tinyagents-harness/src/steering/test.rs b/crates/tinyagents-harness/src/steering/test.rs index 72f1d7ac..d5a51850 100644 --- a/crates/tinyagents-harness/src/steering/test.rs +++ b/crates/tinyagents-harness/src/steering/test.rs @@ -39,6 +39,8 @@ fn text_response(text: &str) -> ModelResponse { )], tool_calls: Vec::new(), usage: Some(Usage::new(1, 1)), + + origin: None, }, usage: Some(Usage::new(1, 1)), finish_reason: Some("stop".to_string()), @@ -87,6 +89,8 @@ impl ChatModel<()> for RecordingModel { content: Vec::new(), tool_calls: vec![ToolCall::new("c1", "noop", json!({}))], usage: Some(Usage::new(1, 1)), + + origin: None, }, usage: Some(Usage::new(1, 1)), finish_reason: Some("tool_calls".to_string()), diff --git a/crates/tinyagents-harness/src/summarization/render.rs b/crates/tinyagents-harness/src/summarization/render.rs index 579dbf7f..454fedaf 100644 --- a/crates/tinyagents-harness/src/summarization/render.rs +++ b/crates/tinyagents-harness/src/summarization/render.rs @@ -41,6 +41,7 @@ fn role_label(message: &Message) -> &'static str { Message::User(_) => "user", Message::Assistant(_) => "assistant", Message::Tool(_) => "tool", + Message::Custom(_) => "custom", } } @@ -76,6 +77,9 @@ fn render_content(content: &[ContentBlock]) -> Vec { "", image.mime_type.as_deref().unwrap_or("unknown") )), + ContentBlock::Audio(_) => Some("