Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -62,10 +62,16 @@ jobs:
cargo test --workspace --no-default-features --features tools
cargo test --workspace --no-default-features --features multimodal

- name: Free build artifacts before coverage
run: cargo clean

- name: Coverage gate
uses: taiki-e/install-action@cargo-llvm-cov

- name: Verify line coverage
env:
CARGO_PROFILE_DEV_DEBUG: 0
CARGO_PROFILE_TEST_DEBUG: 0
run: >-
cargo llvm-cov --all-features --workspace
--ignore-filename-regex '(^|/)(tests?|examples)/|/test(_.*)?\.rs$'
Expand Down
10 changes: 9 additions & 1 deletion crates/tinyagents-harness/src/agent_loop/dialect.rs
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,9 @@ use tinytools_agent::{PFormatRegistry, StreamScrubber};
use crate::config::ToolDispatcher;
use crate::ids::CallId;

#[cfg(test)]
mod test;

/// The dialect a run speaks, resolved once from policy.
#[derive(Debug, Clone)]
pub(super) enum RunDialect {
Expand All @@ -35,8 +38,13 @@ pub(super) enum RunDialect {

impl RunDialect {
/// Resolves the policy against the tools this run offers.
pub(super) fn resolve(dispatcher: ToolDispatcher, tools: &[ToolSchema]) -> Self {
pub(super) fn resolve(
dispatcher: ToolDispatcher,
tools: &[ToolSchema],
native_tool_calling: Option<bool>,
) -> Self {
match dispatcher {
ToolDispatcher::Auto if native_tool_calling == Some(false) => Self::Xml,
Comment thread
senamakel marked this conversation as resolved.
ToolDispatcher::Auto | ToolDispatcher::Native => Self::Native,
ToolDispatcher::Xml => Self::Xml,
ToolDispatcher::Pformat => Self::PFormat(Arc::new(tinytools_agent::build_registry(
Expand Down
17 changes: 17 additions & 0 deletions crates/tinyagents-harness/src/agent_loop/dialect/test.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
//! Focused tests for model-aware tool dialect selection.

use super::RunDialect;
use crate::config::ToolDispatcher;
use tinyinference_llm::model::ModelProfile;

#[test]
fn auto_uses_xml_when_the_model_disables_native_tool_calling() {
let profile = ModelProfile {
tool_calling: false,
..ModelProfile::default()
};

let dialect = RunDialect::resolve(ToolDispatcher::Auto, &[], Some(profile.tool_calling));

assert!(matches!(dialect, RunDialect::Xml));
}
7 changes: 5 additions & 2 deletions crates/tinyagents-harness/src/agent_loop/run_loop.rs
Original file line number Diff line number Diff line change
Expand Up @@ -917,8 +917,11 @@ impl<State: Send + Sync, Ctx: Send + Sync> AgentHarness<State, Ctx> {
// side-effecting call.
let tools_available_this_turn =
offered_tool_count > 0 && request.tool_choice != ToolChoice::None;
let dialect =
super::dialect::RunDialect::resolve(self.policy.tool_dialect, &request.tools);
let dialect = super::dialect::RunDialect::resolve(
self.policy.tool_dialect,
&request.tools,
binding.model.profile().map(|profile| profile.tool_calling),
);
let forced_text_dialect = dialect.is_text();
let recovery = if tools_available_this_turn {
super::dialect::TextRecovery {
Expand Down
1 change: 1 addition & 0 deletions crates/tinyagents-harness/src/agent_loop/test.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5726,6 +5726,7 @@ async fn resolved_profile_schema_transform_is_applied_to_tool_schemas() {

let model = Arc::new(
ScriptedModel::replies(vec!["done"]).with_profile(ModelProfile {
tool_calling: true,
schema_transform: Some(SchemaTransform::StripDefs),
..ModelProfile::default()
}),
Expand Down
14 changes: 13 additions & 1 deletion crates/tinyagents-harness/src/providers/claude_code/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -331,8 +331,20 @@ fn thread_key_from_request(request: &ModelRequest) -> String {
/// provider sends downstream: prompt-tool coalescing/instructions applied
/// first, then any structured [`ResponseFormat`] is appended as a trailing
/// system instruction (see [`response_format_instruction`]).
///
/// `Message::Custom` remains in the host transcript as out-of-band metadata;
/// it is intentionally omitted from the provider prompt. Hosts that want such
/// information available to the model must add it as an explicit system or
/// user message. Its optional `display` text is for transcript presentation,
/// not an implicit prompt channel.
fn request_messages(request: &ModelRequest) -> Vec<ChatMessage> {
let mut messages = coalesce_tool_results(&request.messages);
let provider_messages: Vec<_> = request
.messages
.iter()
.filter(|message| !matches!(message, Message::Custom(_)))
Comment thread
senamakel marked this conversation as resolved.
.cloned()
.collect();
let mut messages = coalesce_tool_results(&provider_messages);
if !request.tools.is_empty() {
messages = with_tool_instructions(&messages, &request.tools, &request.tool_choice);
}
Expand Down
18 changes: 18 additions & 0 deletions crates/tinyagents-harness/src/providers/claude_code/mod_tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -236,3 +236,21 @@ fn request_rendering_keeps_private_image_marker_text_literal() {
assert_eq!(content[0]["text"], "literal ");
assert_eq!(content[1]["text"], "[OH_IMAGE:data:image/png;base64,QUJD]");
}

#[test]
fn request_messages_filter_host_custom_records() {
let request = ModelRequest::new(vec![
Message::user("hello"),
Message::Custom(tinyinference_llm::message::CustomMessage {
kind: "compaction".into(),
payload: serde_json::json!({"summary": "host-only"}),
display: Some("host-only".into()),
}),
]);

let messages = request_messages(&request);

assert_eq!(messages.len(), 1);
assert_eq!(messages[0].role, "user");
assert_eq!(messages[0].content, "hello");
}
9 changes: 9 additions & 0 deletions crates/tinyagents-harness/src/stream/frame.rs
Original file line number Diff line number Diff line change
Expand Up @@ -240,6 +240,9 @@ pub fn encode_frames(items: &[ModelStreamItem]) -> Vec<AssistantFrame> {
enum OpenBlock {
Text(String),
Thinking(String),
ProviderExtension {
block_type: String,
},
ToolCall {
id: Option<String>,
name: Option<String>,
Expand Down Expand Up @@ -317,6 +320,9 @@ pub fn reduce_frames(frames: &[AssistantFrame]) -> PartialAssistantMessage {
let block = match kind {
BlockKind::Text => OpenBlock::Text(String::new()),
BlockKind::Thinking => OpenBlock::Thinking(String::new()),
BlockKind::ProviderExtension { block_type } => OpenBlock::ProviderExtension {
block_type: block_type.clone(),
},
BlockKind::ToolCall { id, name } => OpenBlock::ToolCall {
id: Some(id.clone()),
name: Some(name.clone()),
Expand Down Expand Up @@ -410,6 +416,9 @@ pub fn reduce_frames(frames: &[AssistantFrame]) -> PartialAssistantMessage {
.map(|(index, block)| {
let text = match block {
OpenBlock::Text(text) | OpenBlock::Thinking(text) => text,
OpenBlock::ProviderExtension { block_type } => {
format!("provider extension: {block_type}")
}
OpenBlock::ToolCall { json_so_far, .. } => json_so_far,
};
(index, text)
Expand Down
23 changes: 23 additions & 0 deletions crates/tinyagents-harness/src/stream/frame/test.rs
Original file line number Diff line number Diff line change
Expand Up @@ -283,3 +283,26 @@ fn provider_failed_frame_carries_partial_message_and_stop_reason() {
vec![ContentBlock::Text("partial".into())]
);
}

#[test]
fn provider_extension_frames_retain_the_completed_content_block() {
let extension = json!({"type": "future_block", "opaque": true});
let partial = reduce_frames(&[
AssistantFrame::BlockStart {
index: 0,
kind: BlockKind::ProviderExtension {
block_type: "future_block".into(),
},
},
AssistantFrame::BlockEnd {
index: 0,
block: ContentBlock::ProviderExtension(extension.clone()),
},
]);

assert_eq!(
partial.content,
vec![ContentBlock::ProviderExtension(extension)]
);
assert!(partial.open_blocks.is_empty());
}
6 changes: 3 additions & 3 deletions crates/tinyagents-harness/src/summarization/render.rs
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,9 @@ fn render_content(content: &[ContentBlock]) -> Vec<String> {
"<image mime=\"{}\" />",
image.mime_type.as_deref().unwrap_or("unknown")
)),
ContentBlock::Audio(_) => Some("<audio />".to_string()),
Comment thread
senamakel marked this conversation as resolved.
Comment thread
senamakel marked this conversation as resolved.
ContentBlock::Video(_) => Some("<video />".to_string()),
ContentBlock::Document(_) => Some("<document />".to_string()),
ContentBlock::Thinking { text, .. } if text.trim().is_empty() => None,
ContentBlock::Thinking { text, .. } => {
Some(format!("<reasoning>{}</reasoning>", elide(text)))
Expand All @@ -86,9 +89,6 @@ fn render_content(content: &[ContentBlock]) -> Vec<String> {
"<provider_extension>{}</provider_extension>",
elide(&value.to_string())
)),
ContentBlock::Audio(_) => Some("<audio />".to_string()),
ContentBlock::Video(_) => Some("<video />".to_string()),
ContentBlock::Document(_) => Some("<document />".to_string()),
})
.collect()
}
Expand Down
40 changes: 40 additions & 0 deletions crates/tinyagents-harness/src/summarization/test.rs
Original file line number Diff line number Diff line change
Expand Up @@ -699,6 +699,46 @@ mod rendering {
assert!(rendered.contains("\"k\""), "{rendered}");
}

#[test]
Comment thread
senamakel marked this conversation as resolved.
fn media_blocks_are_rendered_as_placeholders() {
use tinyinference_llm::message::{ContentBlock, MediaRef, UserMessage};

let msg = Message::User(UserMessage {
content: vec![
ContentBlock::Audio(MediaRef::url("https://example.com/a.wav")),
ContentBlock::Video(MediaRef::base64("AAAA", "video/mp4")),
ContentBlock::Document(MediaRef::path("/tmp/doc.pdf")),
],
});
let rendered = render_message_for_summary(&msg);

assert!(rendered.contains("<audio />"), "{rendered}");
assert!(rendered.contains("<video />"), "{rendered}");
assert!(rendered.contains("<document />"), "{rendered}");
}

#[test]
fn custom_messages_render_the_display_or_an_empty_value() {
use tinyinference_llm::message::CustomMessage;

let displayed = Message::Custom(CustomMessage {
kind: "compaction".into(),
payload: json!({"summary": "host-only"}),
display: Some("Compacted 40 turns".into()),
});
let hidden = Message::Custom(CustomMessage {
kind: "label".into(),
payload: json!({"name": "checkpoint"}),
display: None,
});

assert_eq!(
render_message_for_summary(&displayed),
"custom: Compacted 40 turns"
);
assert_eq!(render_message_for_summary(&hidden), "custom: ");
}

#[test]
fn oversized_payloads_are_elided_not_reproduced() {
let msg = Message::tool("c1", "y".repeat(9_000));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -147,11 +147,11 @@ const KNOWN_GENERIC_CLAUDE_CODE_CHAT_MESSAGE_DEBT: &[(&str, usize)] = &[
),
(
"crates/tinyagents-harness/src/providers/claude_code/mod.rs",
334,
340,
),
(
"crates/tinyagents-harness/src/providers/claude_code/mod.rs",
362,
374,
),
(
"crates/tinyagents-harness/src/providers/claude_code/mod_tests.rs",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,7 @@ fn component_metadata_and_event_kinds_are_stable_serializable_contracts() {
let id = ComponentId::new("researcher");
assert_eq!(id.as_str(), "researcher");
assert_eq!(id.to_string(), "researcher");
assert_eq!(ComponentKind::ALL.len(), 11);
assert_eq!(ComponentKind::ALL.len(), 12);
assert_eq!(ComponentKind::Agent.as_str(), "agent");
assert_eq!(ComponentKind::TaskStore.as_str(), "task_store");
assert_eq!(ComponentKind::Tool.to_string(), "tool");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ use tinyagents_harness::config::ToolDispatcher;
use tinyagents_harness::context::RunContext;
use tinyagents_harness::events::{AgentEvent, RecordingListener};
use tinyagents_harness::middleware::Middleware;
use tinyagents_harness::runtime::{AgentHarness, RunPolicy};
use tinyagents_harness::runtime::{AgentHarness, EndStrategy, RunPolicy};
use tinyagents_harness::testkit::{FakeTool, ScriptedModel, StreamingMock};
use tinyinference_llm::message::{Message, MessageDelta};
use tinyinference_llm::model::{
Expand Down Expand Up @@ -118,7 +118,11 @@ async fn a_native_model_narrating_a_call_in_any_grammar_dispatches_it() {
"<tool_call>{\"name\":\"functions.lookup\",\"arguments\":{\"q\":\"x\"}}</tool_call>",
] {
let listener = Arc::new(RecordingListener::new());
let harness = harness_with(Arc::new(narrating_model(text)), &listener);
let mut harness = harness_with(Arc::new(narrating_model(text)), &listener);
harness.with_policy(RunPolicy {
text_dialect_recovery: tinyagents_harness::runtime::TextDialectRecovery::On,
..RunPolicy::default()
});
let run = harness
.invoke_default(&(), vec![Message::user("go")])
.await
Expand Down Expand Up @@ -960,6 +964,7 @@ async fn dropped_call_nudge_budget_resets_after_a_mixed_structured_and_tool_turn
}))
.with_policy(RunPolicy {
dropped_tool_call_nudges: 1,
end_strategy: EndStrategy::Exhaustive,
default_response_format: Some(ResponseFormat::auto(
"answer",
json!({"type": "object"}),
Expand Down
11 changes: 7 additions & 4 deletions crates/tinyagents-orchestration/tests/hosted_subagents.rs
Original file line number Diff line number Diff line change
Expand Up @@ -89,13 +89,15 @@ async fn authorized_child_reuses_the_parent_host_bundle() {
ModelResponse::assistant("parent answer"),
]));
let worker_model = Arc::new(ScriptedModel::replies(vec!["child answer"]));
let parent = AgentDefinition::new("parent", "Parent", "delegates").with_subagents(["worker"]);
let parent = AgentDefinition::new("parent", "Parent", "delegates")
.with_subagents(["worker"])
.with_tools(["worker"]);
let entry = AgentHarness::new();
let (runtime, jobs) = runtime_with_worker(AgentHarness::new());
let run = entry
.invoke_agent(
AgentInvocation::new(
host(parent, parent_model, worker_model),
host(parent, parent_model.clone(), worker_model),
AgentTurnRequest::new(
"parent",
vec![tinyinference_llm::message::Message::user("delegate")],
Expand Down Expand Up @@ -137,7 +139,8 @@ async fn parent_denial_cannot_fall_back_to_the_child_harness() {
.invoke_agent(
AgentInvocation::new(
host(
AgentDefinition::new("parent", "Parent", "does not delegate"),
AgentDefinition::new("parent", "Parent", "does not delegate")
.with_tools(["worker"]),
parent_model,
Arc::new(ScriptedModel::replies(vec!["host worker"])),
),
Expand All @@ -155,7 +158,7 @@ async fn parent_denial_cannot_fall_back_to_the_child_harness() {

assert_eq!(
error.to_string(),
"model error: hosted agent invocation failed"
"hosted agent invocation failed at the model provider"
);
assert!(
local_child_model.requests().is_empty(),
Expand Down
Loading