diff --git a/Cargo.lock b/Cargo.lock index c9e77e61df..7b85ba5fb5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6479,7 +6479,9 @@ dependencies = [ "thiserror 2.0.20", "tinyagents-definition", "tinyinference-embeddings", + "tinyinference-image", "tinyinference-llm", + "tinyinference-video", "tinytools 0.4.1", "tinytools-agent 0.4.1", "tokio", @@ -6787,6 +6789,22 @@ dependencies = [ "url", ] +[[package]] +name = "tinyinference-image" +version = "0.3.0" +dependencies = [ + "async-trait", + "base64 0.23.1", + "bytes", + "reqwest", + "serde", + "serde_json", + "thiserror 2.0.20", + "tinyinference-core", + "tokio", + "tracing", +] + [[package]] name = "tinyinference-llm" version = "0.3.0" @@ -6848,6 +6866,19 @@ dependencies = [ "url", ] +[[package]] +name = "tinyinference-video" +version = "0.3.0" +dependencies = [ + "async-trait", + "serde", + "serde_json", + "thiserror 2.0.20", + "tinyinference-image", + "tokio", + "tracing", +] + [[package]] name = "tinyinference-voice" version = "0.3.0" diff --git a/crates/openhuman-app/Cargo.lock b/crates/openhuman-app/Cargo.lock index 2b39135b2c..757c95cdcb 100644 --- a/crates/openhuman-app/Cargo.lock +++ b/crates/openhuman-app/Cargo.lock @@ -7072,7 +7072,9 @@ dependencies = [ "thiserror 2.0.20", "tinyagents-definition", "tinyinference-embeddings", + "tinyinference-image", "tinyinference-llm", + "tinyinference-video", "tinytools 0.4.1", "tinytools-agent 0.4.1", "tokio", @@ -7410,6 +7412,22 @@ dependencies = [ "url", ] +[[package]] +name = "tinyinference-image" +version = "0.3.0" +dependencies = [ + "async-trait", + "base64 0.23.1", + "bytes", + "reqwest 0.12.28", + "serde", + "serde_json", + "thiserror 2.0.20", + "tinyinference-core", + "tokio", + "tracing", +] + [[package]] name = "tinyinference-llm" version = "0.3.0" @@ -7471,6 +7489,19 @@ dependencies = [ "url", ] +[[package]] +name = "tinyinference-video" +version = "0.3.0" +dependencies = [ + "async-trait", + "serde", + "serde_json", + "thiserror 2.0.20", + "tinyinference-image", + "tokio", + "tracing", +] + [[package]] name = "tinyinference-voice" version = "0.3.0" diff --git a/crates/openhuman-cli/Cargo.toml b/crates/openhuman-cli/Cargo.toml index 079680b8cc..bb66ff0bc5 100644 --- a/crates/openhuman-cli/Cargo.toml +++ b/crates/openhuman-cli/Cargo.toml @@ -251,6 +251,11 @@ path = "../../tests/mcp_registry_multi_server.rs" name = "mcp_stdio_integration" path = "../../tests/mcp_stdio_integration.rs" +[[test]] +name = "media_generation_e2e" +path = "../../tests/media_generation_e2e.rs" +required-features = ["media"] + [[test]] name = "memory_roundtrip_e2e" path = "../../tests/memory_roundtrip_e2e.rs" diff --git a/crates/openhuman-core/Cargo.toml b/crates/openhuman-core/Cargo.toml index 583a22904a..9686077d7b 100644 --- a/crates/openhuman-core/Cargo.toml +++ b/crates/openhuman-core/Cargo.toml @@ -999,16 +999,17 @@ runtime-node = [] # and removing it from one without the other fails that lane. contacts = [] # Media-generation + image domains: the `media_generate_*` agent tools -# (image/video via GMI through the backend) and the `openhuman::image` tool +# (image/video via OpenRouter through the backend's +# `/agent-integrations/openrouter` proxy) and the `openhuman::image` tool # contracts scaffold. Default-ON. Slim builds opt out via # `--no-default-features --features ""`. # Composes with the runtime `DomainSet::media` flag (#4796). -# NOTE: this gate sheds no exclusive dependencies — media generation is -# backend-proxied (reqwest, shared). It is a surface-only gate (drops the tool -# code + module from the compile), not a dependency-shedding one. There are no -# controllers / stores / subscribers tagged `Media` (agent tools only), and -# `openhuman::image` is currently unwired scaffold (added #2997). -media = [] +# Enables `tinyagents-harness/media`, which pulls in `tinyinference-image` and +# `tinyinference-video` (the wire contract, job loop and generic tools); both +# are light (serde + the already-shared reqwest), so this remains mostly a +# surface gate. There are no controllers / stores / subscribers tagged `Media` +# (agent tools only), and `openhuman::image` is currently unwired scaffold. +media = ["tinyagents-harness/media"] # Flows domains: the `flows::` automation surface (saved tinyflows graphs — # create/run/schedule + the workflow_builder / flow_discovery agents), the # `tinyflows::` adapter seam, and the `rhai_workflows::` language-workflow tool. diff --git a/crates/openhuman-core/src/agent/registry/agents/image_agent/agent.toml b/crates/openhuman-core/src/agent/registry/agents/image_agent/agent.toml index e05b471b5a..8e2d853a33 100644 --- a/crates/openhuman-core/src/agent/registry/agents/image_agent/agent.toml +++ b/crates/openhuman-core/src/agent/registry/agents/image_agent/agent.toml @@ -11,10 +11,16 @@ omit_safety_preamble = false omit_profile = true omit_memory_md = true -# Multimodal tier so the agent can review the images it generates (and any -# reference images) via the image_info / inline-image path, then iterate. +# Pinned to a dedicated OpenRouter passthrough model rather than the +# deprecated `hint:vision` (regression R4). This exact id is in +# `MANAGED_MULTIMODAL_MODELS`, so it keeps the multimodal tier needed to +# review the images it generates (and any reference images) via the +# image_info / inline-image path, then iterate. Actual image generation goes +# through `media_generate_image`, which defaults to +# `bytedance-seed/seedream-5-0-lite` on OpenRouter — a separate model from +# the one this agent's own turns run on. [model] -hint = "vision" +exact = "openrouter/qwen/qwen3.7-flash" [tools] # media_generate_image submits the generation and returns a saved local path; diff --git a/crates/openhuman-core/src/agent/registry/agents/image_agent/prompt.md b/crates/openhuman-core/src/agent/registry/agents/image_agent/prompt.md index 5036393a81..e39d67b267 100644 --- a/crates/openhuman-core/src/agent/registry/agents/image_agent/prompt.md +++ b/crates/openhuman-core/src/agent/registry/agents/image_agent/prompt.md @@ -1,28 +1,32 @@ # Image-generation specialist You are a focused **image-creation** sub-agent. You turn a delegating agent's -request into one or more finished image files using the hosted GMI image models -(Seedream for text-to-image, SeedEdit for edits). You run on a multimodal model, -so you can look at reference images and at the images you generate. +request into one or more finished image files using a hosted image-generation +model — the default is `bytedance-seed/seedream-5-0-lite` via OpenRouter for +text-to-image and edits, but the catalog offers other supported models too. +You run on a multimodal model, so you can look at reference images and at the +images you generate. ## Your job - **Create** images from a text prompt (`media_generate_image`). -- **Edit / restyle** a supplied image by passing its URL(s) as `input_images`. -- **Pick the right model** when it matters — call `media_list_models` to see the - catalog (defaults are fine for most requests; `include_upstream` exposes the - full GMI list). +- **Edit / restyle** a supplied image by passing it in `references` (https + URLs, `data:` URLs, or workspace file paths). +- **Pick the right model** when it matters — call `media_list_models` + (`kind: "image"`, optional `search`) to see the catalog. The default suits + most requests. ## How to work - Write a vivid, specific prompt. Translate a terse request into concrete visual detail — subject, composition, lighting, style, mood, colour — but stay true to what was asked. Don't invent requirements the user didn't state. -- Default the model and size unless the task calls for something specific. Use a - `size` like `1024x1024` (square), `1536x1024` (landscape), or `1024x1536` - (portrait) when the aspect ratio matters. -- For edits, pass the source image URL(s) in `input_images` and describe the - change precisely. +- Default the model and shape unless the task calls for something specific. Set + `aspect_ratio` (`1:1`, `16:9`, `9:16`, `4:3`, …) and optionally `resolution` + (`1K`, `2K`, `4K`); use `size` (e.g. `1536x1024`) only when exact pixels + matter. `n` asks for several variants; `seed` makes a result reproducible. +- For edits, pass the source image(s) in `references` and describe the change + precisely. - Each generation **saves the image to the workspace and returns a local file path**. Always report that path back so the deck/answer can reference the concrete artifact. Do not paste raw base64 or invent URLs. @@ -34,5 +38,6 @@ so you can look at reference images and at the images you generate. - Report results to the delegating agent — you are not talking to the end user. - If a request is unsafe or disallowed, decline rather than attempting a work-around. -- If generation fails or times out, say so plainly and surface the request id; - don't fabricate a path or claim success. +- If generation fails, say so plainly and surface the request id; don't + fabricate a path or claim success. When the error says the call was billed, + do **not** call again — report it. diff --git a/crates/openhuman-core/src/agent/registry/agents/loader_tests_builtin_registration_tests.rs b/crates/openhuman-core/src/agent/registry/agents/loader_tests_builtin_registration_tests.rs index 5ba961c614..2ed455b980 100644 --- a/crates/openhuman-core/src/agent/registry/agents/loader_tests_builtin_registration_tests.rs +++ b/crates/openhuman-core/src/agent/registry/agents/loader_tests_builtin_registration_tests.rs @@ -224,12 +224,17 @@ fn every_builtin_is_stamped_builtin_source() { } #[test] -fn vision_agent_loads_on_vision_hint() { - // The vision sub-agent rides the multimodal `vision-v1` tier (via the - // `vision` hint) so its model is image-capable, and it must be reachable - // from the orchestrator's subagent allowlist. +fn vision_agent_loads_on_its_pinned_multimodal_model() { + // The vision sub-agent used to ride the multimodal `vision-v1` tier (via + // the `vision` hint), which is now deprecated — `vision-v1` silently + // falls back to the chat default on managed routes (regression R4). It + // is pinned to a dedicated OpenRouter passthrough model instead, and + // must remain reachable from the orchestrator's subagent allowlist. let def = find("vision_agent"); - assert!(matches!(def.model, ModelSpec::Hint(ref h) if h == "vision")); + assert!(matches!( + def.model, + ModelSpec::Exact(ref m) if m == crate::config::MODEL_MEDIA_UNDERSTANDING + )); let orchestrator = find("orchestrator"); assert!( diff --git a/crates/openhuman-core/src/agent/registry/agents/loader_tests_specialist_agents_tests.rs b/crates/openhuman-core/src/agent/registry/agents/loader_tests_specialist_agents_tests.rs index 6515bb21a1..ae845d2ac5 100644 --- a/crates/openhuman-core/src/agent/registry/agents/loader_tests_specialist_agents_tests.rs +++ b/crates/openhuman-core/src/agent/registry/agents/loader_tests_specialist_agents_tests.rs @@ -528,6 +528,44 @@ fn code_executor_has_curl_for_artifact_downloads() { } } +/// R4 regression: `hint:vision` is deprecated (`vision-v1` silently falls +/// back to the chat default on managed routes, with no error), so no +/// built-in agent may still declare `ModelSpec::Hint("vision")`. +#[test] +fn no_builtin_agent_declares_the_deprecated_vision_hint() { + for def in load_builtins().expect("built-ins load") { + assert!( + !matches!(&def.model, ModelSpec::Hint(h) if h == "vision"), + "`{}` still declares the deprecated `hint:vision` — pin an exact model instead", + def.id + ); + } +} + +/// The three media agents are pinned to their dedicated OpenRouter +/// passthrough models (regression R4), not left on `Inherit` or a `Hint`. +#[test] +fn media_agents_are_pinned_to_their_exact_models() { + use crate::config::{ + MODEL_IMAGE_GENERATION_AGENT, MODEL_MEDIA_UNDERSTANDING, MODEL_VIDEO_GENERATION_AGENT, + }; + + for (agent_id, expected_model) in [ + ("vision_agent", MODEL_MEDIA_UNDERSTANDING), + ("image_agent", MODEL_IMAGE_GENERATION_AGENT), + ("video_agent", MODEL_VIDEO_GENERATION_AGENT), + ] { + let def = find(agent_id); + match &def.model { + ModelSpec::Exact(model) => assert_eq!( + model, expected_model, + "{agent_id} must be pinned to `{expected_model}`, got `{model}`" + ), + other => panic!("{agent_id} must use ModelSpec::Exact, got {other:?}"), + } + } +} + #[test] fn orchestrator_does_not_get_curl() { // Per design: curl is a `Write` permission tool that writes diff --git a/crates/openhuman-core/src/agent/registry/agents/orchestrator/agent.toml b/crates/openhuman-core/src/agent/registry/agents/orchestrator/agent.toml index fe053e2e1f..9a5df65c45 100644 --- a/crates/openhuman-core/src/agent/registry/agents/orchestrator/agent.toml +++ b/crates/openhuman-core/src/agent/registry/agents/orchestrator/agent.toml @@ -121,8 +121,9 @@ allowlist = [ # the memory tree only when a message needs it, not before every turn. "agent_memory", # Image-understanding specialist. Route anything that hinges on the content - # of an attached or on-disk user-provided image file here — it rides the - # multimodal `hint:vision` tier, so it can actually see the image. + # of an attached or on-disk user-provided image file here — it is pinned to + # a dedicated multimodal model (`MODEL_MEDIA_UNDERSTANDING`), so it can + # actually see the image. "vision_agent", # Image-generation specialist. Synthesised into a `delegate_create_image` # tool. Route make/generate/edit an image requests here — it owns prompt diff --git a/crates/openhuman-core/src/agent/registry/agents/video_agent/agent.toml b/crates/openhuman-core/src/agent/registry/agents/video_agent/agent.toml index 07568ca247..80be1702c1 100644 --- a/crates/openhuman-core/src/agent/registry/agents/video_agent/agent.toml +++ b/crates/openhuman-core/src/agent/registry/agents/video_agent/agent.toml @@ -11,10 +11,15 @@ omit_safety_preamble = false omit_profile = true omit_memory_md = true -# Multimodal tier so the agent can inspect a reference/first-frame image or the -# returned thumbnail when shaping an image-to-video request. +# Pinned to a dedicated OpenRouter passthrough model rather than the +# deprecated `hint:vision` (regression R4). This exact id is in +# `MANAGED_MULTIMODAL_MODELS`, so it keeps the multimodal tier needed to +# inspect a reference/first-frame image or the returned thumbnail when +# shaping an image-to-video request. Actual video generation goes through +# `media_generate_video`, which defaults to `bytedance/seedance-2.0-mini` on +# OpenRouter — a separate model from the one this agent's own turns run on. [model] -hint = "vision" +exact = "openrouter/qwen/qwen3.7-flash" [tools] # media_generate_video submits the generation and returns a saved local path; diff --git a/crates/openhuman-core/src/agent/registry/agents/video_agent/prompt.md b/crates/openhuman-core/src/agent/registry/agents/video_agent/prompt.md index 97d6a7c76b..dc622c8162 100644 --- a/crates/openhuman-core/src/agent/registry/agents/video_agent/prompt.md +++ b/crates/openhuman-core/src/agent/registry/agents/video_agent/prompt.md @@ -1,27 +1,33 @@ # Video-generation specialist You are a focused **video-creation** sub-agent. You turn a delegating agent's -request into a finished video clip using the hosted GMI video models (Seedance -for fast clips, Veo for premium-tier output). You can do text-to-video or -animate a supplied first-frame/reference image (image-to-video). +request into a finished video clip using a hosted video-generation model — the +default is `bytedance/seedance-2.0-mini` via OpenRouter for fast clips, with +premium-tier models available in the catalog for higher-quality output. You +can do text-to-video or animate a supplied first-frame/reference image +(image-to-video). ## Your job - **Create** a clip from a text prompt (`media_generate_video`). -- **Animate** a supplied image by passing its URL as `input_image`. -- **Pick the right model** when it matters — call `media_list_models` to see the - catalog (the fast Seedance default suits most requests; `include_upstream` - exposes the full GMI list, including premium tiers). +- **Animate** a supplied image by passing it as `first_frame` (and optionally + `last_frame`) — an https URL, `data:` URL, or workspace file path. +- **Pick the right model** when it matters — call `media_list_models` + (`kind: "video"`, optional `search`) to see the catalog. The fast default + suits most requests. ## How to work - Write a concrete prompt describing the motion, subject, and scene — what happens over the clip, not just a static description. Mention camera movement, pacing, and style when relevant. -- Use `duration_seconds` and `aspect_ratio` (e.g. `16:9`, `9:16`, `1:1`) when the - task specifies them; otherwise let the model default. -- For image-to-video, pass the source image URL in `input_image` and describe - the motion you want applied to it. +- Use `duration` (seconds; the default model accepts 4–15), `aspect_ratio` + (e.g. `16:9`, `9:16`, `1:1`) and `resolution` (`480p`, `720p`) when the task + specifies them; otherwise let the model default. `generate_audio` adds a + soundtrack where supported. +- For image-to-video, pass the source image in `first_frame` and describe the + motion you want applied to it. `references` guide subject or style without + fixing a frame. - Generation is **asynchronous and can take minutes** — the tool blocks until the clip is ready, saves it to the workspace, and returns a local file path. Report that path back. Set expectations: tell the delegating agent it may take a @@ -34,5 +40,7 @@ animate a supplied first-frame/reference image (image-to-video). - Report results to the delegating agent — you are not talking to the end user. - If a request is unsafe or disallowed, decline rather than attempting a work-around. -- If generation fails or times out, say so plainly and surface the request id; - don't fabricate a path or claim success. +- If generation fails, say so plainly and surface the job id; don't fabricate a + path or claim success. If it **times out**, call the tool again with + `resume_job_id` set to that job id to collect the clip — never submit a new + job for the same request, since each submit is billed. diff --git a/crates/openhuman-core/src/agent/registry/agents/vision_agent/agent.toml b/crates/openhuman-core/src/agent/registry/agents/vision_agent/agent.toml index 60d6c01de1..c857a01b73 100644 --- a/crates/openhuman-core/src/agent/registry/agents/vision_agent/agent.toml +++ b/crates/openhuman-core/src/agent/registry/agents/vision_agent/agent.toml @@ -10,12 +10,14 @@ omit_identity = true omit_memory_context = true omit_safety_preamble = true -# Multimodal tier. `ModelSpec::Hint("vision")` resolves to `hint:vision`, which -# `oh_tier_supports_vision` reports as vision-capable — so this sub-agent's -# model is always treated as image-enabled (managed or BYOK), and the turn -# engine never strips the attached image at the vision gate. +# Pinned to a dedicated OpenRouter passthrough model rather than the +# deprecated `hint:vision` (`vision-v1` silently falls back to the chat +# default on managed routes, with no error — regression R4). This exact id +# is in `MANAGED_MULTIMODAL_MODELS`, so `oh_tier_supports_vision` still +# reports it as image-enabled and the turn engine never strips the attached +# image at the vision gate. [model] -hint = "vision" +exact = "openrouter/qwen/qwen3.7-flash" [tools] # Attached images arrive inline in the sub-agent's context via the multimodal diff --git a/crates/openhuman-core/src/agent/tinyagents/harness_tool_registration.rs b/crates/openhuman-core/src/agent/tinyagents/harness_tool_registration.rs index 29d2724937..997e955ebf 100644 --- a/crates/openhuman-core/src/agent/tinyagents/harness_tool_registration.rs +++ b/crates/openhuman-core/src/agent/tinyagents/harness_tool_registration.rs @@ -7,6 +7,7 @@ use std::collections::HashSet; use std::sync::Arc; use tinyagents_harness::runtime::AgentHarness; +use tinyagents_harness::tool::ToolDispatch; use tinyagents_registry::{ CapabilityRegistry, ComponentKind, RegistryDiagnostic, RegistrySnapshot, }; @@ -20,8 +21,52 @@ use crate::agent::orchestration::tools::{ use crate::agent::tinyagents::host::OpenHumanRunContext; use crate::agent::tinyagents::tools::{CanonicalSharedToolAdapter, EarlyExitHook}; use crate::agent::tinyagents::turn_policy::is_subagent_spawn_or_delegate_tool; +use crate::agent::tinyagents::use_skill_dispatch::UseSkillDispatch; use crate::agent::tools::{DelegateToolDispatch, TodoToolDispatch}; use crate::memory::agent::CallMemoryAgentDispatch; +use crate::tools::toolpacks::USE_SKILL; + +/// Typed-dispatch selection shared by the direct per-turn registration below +/// and by [`UseSkillDispatch`], which must resolve the SAME live-parent +/// dispatch for a packed archetype delegation (`create_image`, `do_crypto`, +/// `make_presentation`, …) reached through `use_skill` instead of natively +/// advertised (regression R3: `use_skill` used to hand every packed tool to +/// plain `Tool::execute_with_context`, which has no live parent, so a packed +/// delegation always failed with "delegation requires a live harness run +/// context."). +/// +/// `adapter` is expected to be the same `CanonicalSharedToolAdapter` seam +/// used at registration: dispatch selection keys off `name` and the tool's +/// own schema (via [`DelegationDispatch::for_tool`]'s fallback), not object +/// identity, so a freshly built adapter over the resolved tool's registry +/// slot is equivalent to the one the harness itself would have registered. +pub(crate) fn typed_dispatch_for( + name: &str, + adapter: Arc, +) -> Option>> { + let dispatch: Arc> = match name { + "spawn_parallel_agents" => Arc::new(SpawnParallelAgentsDispatch::new(adapter)), + "spawn_async_subagent" => Arc::new(SpawnAsyncSubagentDispatch::new(adapter)), + "spawn_worker_thread" => Arc::new(SpawnWorkerThreadDispatch::new(adapter)), + "spawn_subagent" => Arc::new(SpawnSubagentDispatch::new(adapter)), + "continue_subagent" => Arc::new(ContinueSubagentDispatch::new(adapter)), + "wait_subagent" => Arc::new(WaitSubagentDispatch::new(adapter)), + "steer_subagent" => Arc::new(SteerSubagentDispatch::new(adapter)), + "close_subagent" => Arc::new(CloseSubagentDispatch::new(adapter)), + "list_subagents" => Arc::new(ListSubagentsDispatch::new(adapter)), + "agent_prepare_context" => Arc::new(AgentPrepareContextDispatch::new(adapter)), + "delegate_graph" => Arc::new(DelegateGraphDispatch::new(adapter)), + "delegate" => Arc::new(DelegateToolDispatch::new(adapter)), + "todo" => Arc::new(TodoToolDispatch::new(adapter)), + "call_memory_agent" => Arc::new(CallMemoryAgentDispatch::new(adapter)), + _ => { + return DelegationDispatch::for_tool(adapter).map(|dispatch| { + Arc::new(dispatch) as Arc> + }) + } + }; + Some(dispatch) +} /// Register every admitted tool from `tool_sets` onto `harness` (and its /// `capability_registry` projection), project the visible agent set as @@ -99,43 +144,31 @@ pub(super) fn register_turn_tools_and_agents( registered.insert(name.to_string()); let adapter = Arc::new(adapter); capability_registry.replace_tool(adapter.clone()); - if name == "spawn_parallel_agents" { - harness.register_tool_dispatch(Arc::new(SpawnParallelAgentsDispatch::new( - adapter, - ))); - } else if name == "spawn_async_subagent" { - harness - .register_tool_dispatch(Arc::new(SpawnAsyncSubagentDispatch::new(adapter))); - } else if name == "spawn_worker_thread" { - harness - .register_tool_dispatch(Arc::new(SpawnWorkerThreadDispatch::new(adapter))); - } else if name == "spawn_subagent" { - harness.register_tool_dispatch(Arc::new(SpawnSubagentDispatch::new(adapter))); - } else if name == "continue_subagent" { - harness - .register_tool_dispatch(Arc::new(ContinueSubagentDispatch::new(adapter))); - } else if name == "wait_subagent" { - harness.register_tool_dispatch(Arc::new(WaitSubagentDispatch::new(adapter))); - } else if name == "steer_subagent" { - harness.register_tool_dispatch(Arc::new(SteerSubagentDispatch::new(adapter))); - } else if name == "close_subagent" { - harness.register_tool_dispatch(Arc::new(CloseSubagentDispatch::new(adapter))); - } else if name == "list_subagents" { - harness.register_tool_dispatch(Arc::new(ListSubagentsDispatch::new(adapter))); - } else if name == "agent_prepare_context" { - harness.register_tool_dispatch(Arc::new(AgentPrepareContextDispatch::new( - adapter, - ))); - } else if name == "delegate_graph" { - harness.register_tool_dispatch(Arc::new(DelegateGraphDispatch::new(adapter))); - } else if name == "delegate" { - harness.register_tool_dispatch(Arc::new(DelegateToolDispatch::new(adapter))); - } else if name == "todo" { - harness.register_tool_dispatch(Arc::new(TodoToolDispatch::new(adapter))); - } else if name == "call_memory_agent" { - harness.register_tool_dispatch(Arc::new(CallMemoryAgentDispatch::new(adapter))); - } else if let Some(dispatch) = DelegationDispatch::for_tool(adapter.clone()) { - harness.register_tool_dispatch(Arc::new(dispatch)); + if name == USE_SKILL { + // `use_skill` needs its own typed dispatch (regression + // R3): it is the proxy every packed archetype delegation + // (`create_image`, `do_crypto`, `make_presentation`, …) + // is reached through, and it must resolve the SAME live + // parent `typed_dispatch_for` gives a natively advertised + // delegate tool. The pack-registry handle comes off the + // raw registered tool (not this adapter, which has no + // erased host extension of its own). + let handle = tool_sets + .iter() + .flat_map(|set| set.iter()) + .find(|tool| tool.name() == name) + .and_then(|tool| { + crate::tools::host_extensions::pack_registry_handle(tool.as_ref()) + }) + .cloned(); + match handle { + Some(handle) => harness.register_tool_dispatch(Arc::new( + UseSkillDispatch::new(adapter, handle), + )), + None => harness.register_tool(adapter), + }; + } else if let Some(dispatch) = typed_dispatch_for(name, adapter.clone()) { + harness.register_tool_dispatch(dispatch); } else { harness.register_tool(adapter); } diff --git a/crates/openhuman-core/src/agent/tinyagents/mod.rs b/crates/openhuman-core/src/agent/tinyagents/mod.rs index 1c816f64c6..673e66ef6c 100644 --- a/crates/openhuman-core/src/agent/tinyagents/mod.rs +++ b/crates/openhuman-core/src/agent/tinyagents/mod.rs @@ -60,6 +60,7 @@ mod turn_policy; mod turn_run_error; mod turn_run_finalize; mod turn_runner; +mod use_skill_dispatch; pub(crate) use crate::agent::message_convert::chat_message_to_message; #[cfg(feature = "flows")] diff --git a/crates/openhuman-core/src/agent/tinyagents/use_skill_dispatch.rs b/crates/openhuman-core/src/agent/tinyagents/use_skill_dispatch.rs new file mode 100644 index 0000000000..ec70e4acb2 --- /dev/null +++ b/crates/openhuman-core/src/agent/tinyagents/use_skill_dispatch.rs @@ -0,0 +1,137 @@ +//! Typed dispatch for `use_skill` (regression R3). +//! +//! `use_skill` is registered as a plain canonical tool, so a call reaching a +//! packed archetype delegation (`create_image`, `do_crypto`, +//! `make_presentation`, …) through it used to run through +//! `tinytools::Tool::execute_with_context`, which has no live parent +//! `RunContext`. `dispatch_subagent_with_live_parent` refuses to run without +//! one — "delegation requires a live harness run context." — so every packed +//! delegate was reachable only when a model happened to call it under its +//! bare name, and `PackedToolRouteMiddleware` actively rewrites bare packed +//! calls INTO `use_skill`, making the bug unconditional for any tool this +//! build packs. +//! +//! [`UseSkillDispatch`] closes the gap: it resolves the inner tool exactly as +//! [`crate::tools::toolpacks::tools::UseSkillTool::execute_with_context`] +//! does, then re-selects the same +//! [`super::harness_tool_registration::typed_dispatch_for`] the harness would +//! have picked had the inner tool been natively advertised, and hands it the +//! REAL parent this dispatch itself received. The disclosure half (no `tool` +//! named), a missing `skill`, and a not-found tool are all delegated verbatim +//! to the wrapped `use_skill` adapter — those paths render `UseSkillTool`'s +//! existing schema listing / error text and need no live parent, so +//! duplicating that logic here would only create a second place for it to +//! drift. + +use std::sync::Arc; + +use async_trait::async_trait; +use serde_json::Value; +use tinyagents_harness::context::RunContext; +use tinyagents_harness::tool::{ToolDispatch, ToolExecutionContext}; +use tinytools::{Tool, ToolCallOptions, ToolResult}; + +use super::harness_tool_registration::typed_dispatch_for; +use super::host::OpenHumanRunContext; +use crate::tools::toolpacks::{named_tool, PackRegistryHandle}; + +/// Live-parent dispatch for the `use_skill` proxy tool. +/// +/// `tool` is the `CanonicalSharedToolAdapter` the harness registered for +/// `use_skill` itself (used for the disclosure/error fallback paths and for +/// `tool()`); `handle` is the same pack-registry handle `use_skill`'s own +/// tool object carries, read off its erased host extension at registration +/// time. +pub(crate) struct UseSkillDispatch { + tool: Arc, + handle: PackRegistryHandle, +} + +impl UseSkillDispatch { + pub(crate) fn new(tool: Arc, handle: PackRegistryHandle) -> Self { + Self { tool, handle } + } +} + +#[async_trait] +impl ToolDispatch<(), OpenHumanRunContext> for UseSkillDispatch { + fn tool(&self) -> Arc { + self.tool.clone() + } + + async fn execute( + &self, + _state: &(), + call_id: tinyagents_harness::CallId, + arguments: Value, + options: ToolCallOptions, + parent: &RunContext, + ) -> anyhow::Result { + // Resolve `skill` + `tool` against the pack registry exactly as + // `UseSkillTool::execute_with_context` does. Anything that does not + // resolve here — no `skill`, the disclosure half (no `tool` named), + // or a name the pack does not own — is a path that tool already + // renders correctly and that touches no live parent, so fall through + // to it verbatim rather than re-deriving the same schema listing or + // not-found message. + let resolved = arguments + .get("skill") + .and_then(Value::as_str) + .zip(named_tool(&arguments)) + .and_then(|(skill, name)| { + self.handle + .resolve_registry_for(skill, name) + .map(|tools| (name.to_string(), tools)) + }); + + let Some((name, tools)) = resolved else { + return self + .tool + .execute_with_context(arguments, options, None) + .await; + }; + + let inner_args = arguments + .get("args") + .cloned() + .unwrap_or_else(|| serde_json::json!({})); + + // Re-wrap the resolved tool in the same `CanonicalSharedToolAdapter` + // seam the harness itself builds at registration: typed-dispatch + // selection keys off the tool's name and schema + // (`DelegationDispatch::for_tool`), not object identity, so this + // adapter is equivalent to the one that would have been registered + // had the model reached `name` directly instead of through + // `use_skill`. + let Some(inner_adapter) = + super::tools::CanonicalSharedToolAdapter::for_name(vec![tools], &name) + .map(|adapter| Arc::new(adapter) as Arc) + else { + return self + .tool + .execute_with_context(arguments, options, None) + .await; + }; + + if let Some(dispatch) = typed_dispatch_for(&name, inner_adapter.clone()) { + return dispatch + .execute(&(), call_id, inner_args, options, parent) + .await; + } + + // Not a typed-dispatch tool: run it the way `use_skill` always has, + // through `Tool::execute_with_context`, but still hand it a real + // `ToolExecutionContext` built from the live parent rather than + // `None` — a non-recursive packed tool that reads call id, thread id + // or workspace off the erased host extension gets the same facts a + // native registration would have given it. + let tool_context = ToolExecutionContext::from_run_context(parent, call_id); + inner_adapter + .execute_with_context(inner_args, options, Some(&tool_context)) + .await + } +} + +#[cfg(test)] +#[path = "use_skill_dispatch_tests.rs"] +mod tests; diff --git a/crates/openhuman-core/src/agent/tinyagents/use_skill_dispatch_tests.rs b/crates/openhuman-core/src/agent/tinyagents/use_skill_dispatch_tests.rs new file mode 100644 index 0000000000..5a5c76a8b2 --- /dev/null +++ b/crates/openhuman-core/src/agent/tinyagents/use_skill_dispatch_tests.rs @@ -0,0 +1,284 @@ +//! Regression coverage for R3: `use_skill` must reach a packed archetype +//! delegation (`create_image`, `do_crypto`, `make_presentation`, …) through +//! the SAME live-parent typed dispatch a natively advertised delegate tool +//! gets, not the plain `Tool::execute_with_context` path that has no parent +//! to recurse into. +//! +//! Before this fix, [`UseSkillDispatch`] did not exist and `use_skill` was +//! registered with `harness.register_tool(adapter)` — a plain registration +//! that always calls `Tool::execute_with_context(.., None)`. Reaching +//! `create_image` (the `image_agent` archetype delegate) through `use_skill` +//! therefore always failed with "delegation requires a live harness run +//! context." even when the model's actual turn had one. Reverting +//! `use_skill`'s registration to `harness.register_tool(adapter)` reproduces +//! that failure and is the fastest way to see these tests fail red. + +use super::*; +use crate::agent::harness::definition::AgentDefinitionRegistry; +use crate::agent::harness::ParentExecutionContext; +use crate::agent::prompts::ToolCallFormat; +use crate::agent::tinyagents::tools::CanonicalSharedToolAdapter; +use crate::memory::{Memory, MemoryCategory, MemoryEntry, NamespaceSummary, RecallOpts}; +use crate::tools::toolpacks::tools::{PackRegistryHandle, UseSkillTool}; +use async_trait::async_trait; +use serde_json::json; +use std::path::Path; +use std::sync::Arc; +use tinyagents_harness::context::RunConfig; +use tinyagents_harness::CallId; +use tinyinference_llm::message::Message; +use tinyinference_llm::model::{ChatModel, ModelProfile, ModelRequest, ModelResponse}; +use tinytools::{Tool, ToolCallOptions, ToolResult}; + +/// A stub archetype-delegate tool: only its name has to match `image_agent`'s +/// `delegate_name` ("create_image", set in +/// `agent/registry/agents/image_agent/agent.toml`) for +/// `DelegationDispatch::for_tool` to select the archetype path. Real +/// archetype dispatch never calls the wrapped tool's own `execute` — it +/// dispatches straight to `execute_archetype_delegation_with_live_parent` — +/// so this stub's body is unreachable in a passing run. +struct StubCreateImage; + +#[async_trait] +impl Tool for StubCreateImage { + fn name(&self) -> &str { + "create_image" + } + fn description(&self) -> &str { + "stub archetype delegate for image_agent" + } + fn parameters_schema(&self) -> serde_json::Value { + json!({}) + } + async fn execute(&self, _args: serde_json::Value) -> anyhow::Result { + unreachable!("archetype dispatch must not fall back to the wrapped tool's own execute") + } +} + +/// A `ChatModel` that answers any request, so a real (short) sub-agent turn +/// can run to completion without a network dependency or a canary match. +struct AnyAnswerModel; + +#[async_trait] +impl ChatModel<()> for AnyAnswerModel { + fn profile(&self) -> Option<&ModelProfile> { + static PROFILE: std::sync::OnceLock = std::sync::OnceLock::new(); + Some(PROFILE.get_or_init(|| { + let mut profile = ModelProfile::default(); + profile.tool_calling = true; + profile + })) + } + + async fn invoke( + &self, + _state: &(), + _request: ModelRequest, + ) -> tinyinference_llm::Result { + Ok(ModelResponse::assistant("a generated image description")) + } +} + +#[allow(dead_code)] +fn unused_message_ref(_m: &Message) {} + +struct NoopMemory; + +#[async_trait] +impl Memory for NoopMemory { + async fn store( + &self, + _namespace: &str, + _key: &str, + _value: &str, + _category: MemoryCategory, + _source: Option<&str>, + ) -> anyhow::Result<()> { + Ok(()) + } + + async fn recall( + &self, + _query: &str, + _limit: usize, + _opts: RecallOpts<'_>, + ) -> anyhow::Result> { + Ok(Vec::new()) + } + + async fn get(&self, _namespace: &str, _key: &str) -> anyhow::Result> { + Ok(None) + } + + async fn list( + &self, + _namespace: Option<&str>, + _category: Option<&MemoryCategory>, + _source: Option<&str>, + ) -> anyhow::Result> { + Ok(Vec::new()) + } + + async fn forget(&self, _namespace: &str, _key: &str) -> anyhow::Result { + Ok(false) + } + + async fn namespace_summaries(&self) -> anyhow::Result> { + Ok(Vec::new()) + } + + async fn count(&self) -> anyhow::Result { + Ok(0) + } + + async fn health_check(&self) -> bool { + true + } + + fn name(&self) -> &str { + "noop" + } +} + +fn parent_execution_context(workspace_dir: &Path) -> ParentExecutionContext { + ParentExecutionContext { + workspace_descriptor: None, + agent_definition_id: "orchestrator".into(), + allowed_subagent_ids: ["image_agent".to_string()].into_iter().collect(), + turn_model_source: crate::agent::tinyagents::TurnModelSource::from_model(Arc::new( + AnyAnswerModel, + )), + all_tools: Arc::new(Vec::new()), + all_tool_specs: Arc::new(Vec::new()), + visible_tool_specs: Arc::new(Vec::new()), + visible_tool_names: std::collections::HashSet::new(), + subagent_tool_ceiling_names: std::collections::HashSet::new(), + model_name: "test-model".into(), + temperature: 0.2, + workspace_dir: workspace_dir.to_path_buf(), + memory: Arc::new(NoopMemory), + agent_config: Default::default(), + workflows: Arc::new(Vec::new()), + memory_context: Arc::new(None), + session_id: "use-skill-dispatch-tests".into(), + channel: "test".into(), + connected_integrations: Vec::new(), + tool_call_format: ToolCallFormat::Native, + session_key: "use-skill-dispatch-tests".into(), + session_parent_prefix: None, + on_progress: None, + run_queue: None, + } +} + +/// Builds the `use_skill` registration exactly as +/// `register_turn_tools_and_agents` does: a durable registry containing the +/// real `UseSkillTool` plus one packed archetype-delegate stub, a +/// `PackRegistryHandle` bound to it, and the `CanonicalSharedToolAdapter` +/// wrapping `use_skill` that the harness would have registered. +fn build_use_skill_dispatch() -> UseSkillDispatch { + let handle = PackRegistryHandle::default(); + let use_skill_tool: Box = Box::new(UseSkillTool::new(handle.clone())); + let create_image_tool: Box = Box::new(StubCreateImage); + let durable: Arc>> = Arc::new(vec![use_skill_tool, create_image_tool]); + handle.bind(Arc::downgrade(&durable)); + + let adapter = + CanonicalSharedToolAdapter::for_name(vec![durable], crate::tools::toolpacks::USE_SKILL) + .expect("use_skill resolves in the durable registry it was just placed in"); + UseSkillDispatch::new(Arc::new(adapter), handle) +} + +/// The regression itself: `use_skill { skill: "media", tool: "create_image", +/// args: { prompt: "x", blocking: true } }` must reach the live-parent +/// archetype-delegation path instead of failing with "delegation requires a +/// live harness run context." +#[tokio::test] +async fn use_skill_dispatch_reaches_live_parent_for_packed_archetype_delegate() { + let _ = AgentDefinitionRegistry::init_global_builtins(); + let dispatch = build_use_skill_dispatch(); + let workspace = tempfile::TempDir::new().expect("workspace"); + + let parent_data = crate::agent::tinyagents::host::OpenHumanRunContext::new() + .with_parent(parent_execution_context(workspace.path())); + let parent = parent_data.into_tinyagents(RunConfig::new("use-skill-dispatch-parent")); + + let result = dispatch + .execute( + &(), + CallId::new("use-skill-call"), + json!({ + "skill": "media", + "tool": "create_image", + "args": { "prompt": "a red bicycle", "blocking": true }, + }), + ToolCallOptions::default(), + &parent, + ) + .await + .expect("dispatch returns a tool result rather than an Err"); + + let output = result.output(); + assert!( + !output.contains("requires a live harness run context"), + "use_skill must hand the packed delegation its live parent, not fail on the \ + standalone-caller guard: {output}" + ); +} + +/// The disclosure half (no `tool` named) needs no live parent at all, and +/// must keep behaving exactly like `UseSkillTool::execute_with_context` — +/// [`UseSkillDispatch`] delegates to it rather than re-deriving the listing. +#[tokio::test] +async fn use_skill_dispatch_disclosure_half_delegates_to_use_skill_tool() { + let dispatch = build_use_skill_dispatch(); + let workspace = tempfile::TempDir::new().expect("workspace"); + let parent_data = crate::agent::tinyagents::host::OpenHumanRunContext::new(); + let parent = parent_data.into_tinyagents(RunConfig::new("use-skill-disclosure-parent")); + let _ = workspace; // keep the temp dir alive for symmetry with the other test + + let result = dispatch + .execute( + &(), + CallId::new("use-skill-disclosure-call"), + json!({ "skill": "media" }), + ToolCallOptions::default(), + &parent, + ) + .await + .expect("disclosure half returns a tool result"); + + assert!(!result.is_error, "{}", result.output()); + assert!( + result.output().contains("create_image"), + "the rendered pack listing must include the bound tool: {}", + result.output() + ); +} + +/// A `tool` the pack does not own (or that is not bound) is the not-found +/// path, and also needs no live parent. +#[tokio::test] +async fn use_skill_dispatch_unknown_tool_reports_not_found() { + let dispatch = build_use_skill_dispatch(); + let parent_data = crate::agent::tinyagents::host::OpenHumanRunContext::new(); + let parent = parent_data.into_tinyagents(RunConfig::new("use-skill-not-found-parent")); + + let result = dispatch + .execute( + &(), + CallId::new("use-skill-not-found-call"), + json!({ "skill": "media", "tool": "not_a_real_tool" }), + ToolCallOptions::default(), + &parent, + ) + .await + .expect("not-found half returns a tool result"); + + assert!(result.is_error); + assert!( + result.output().contains("not_a_real_tool"), + "{}", + result.output() + ); +} diff --git a/crates/openhuman-core/src/config/mod.rs b/crates/openhuman-core/src/config/mod.rs index e274a560c3..86240f0c9a 100644 --- a/crates/openhuman-core/src/config/mod.rs +++ b/crates/openhuman-core/src/config/mod.rs @@ -58,9 +58,10 @@ pub use schema::{ TelegramConfig, TokenjuiceConfig, UpdateConfig, UpdateRestartStrategy, VoiceActivationMode, VoiceServerConfig, WebSearchConfig, WebhookConfig, YuanbaoConfig, DEFAULT_CLOUD_LLM_MODEL, DEFAULT_MEMORY_SYNC_INTERVAL_SECS, DEFAULT_MODEL, LEGACY_TIER_MODELS, - MEMORY_SYNC_INTERVAL_PRESETS_SECS, MODEL_MANAGED_DEFAULT, SEARCH_ENGINE_BRAVE, - SEARCH_ENGINE_DISABLED, SEARCH_ENGINE_EXA, SEARCH_ENGINE_MANAGED, SEARCH_ENGINE_PARALLEL, - SEARCH_ENGINE_QUERIT, SEARCH_ENGINE_TAVILY, + MANAGED_MULTIMODAL_MODELS, MEMORY_SYNC_INTERVAL_PRESETS_SECS, MODEL_IMAGE_GENERATION_AGENT, + MODEL_MANAGED_DEFAULT, MODEL_MEDIA_UNDERSTANDING, MODEL_VIDEO_GENERATION_AGENT, + SEARCH_ENGINE_BRAVE, SEARCH_ENGINE_DISABLED, SEARCH_ENGINE_EXA, SEARCH_ENGINE_MANAGED, + SEARCH_ENGINE_PARALLEL, SEARCH_ENGINE_QUERIT, SEARCH_ENGINE_TAVILY, }; // Kept as a separate re-export (issue #4117) so the large alphabetized group // above stays byte-identical and rustfmt-stable. diff --git a/crates/openhuman-core/src/config/schema/types.rs b/crates/openhuman-core/src/config/schema/types.rs index 315e745f81..3b0039e5c3 100644 --- a/crates/openhuman-core/src/config/schema/types.rs +++ b/crates/openhuman-core/src/config/schema/types.rs @@ -12,7 +12,9 @@ mod resolvers; pub use config::{Config, CustomEmbeddingsConfig, ModelRegistryEntry}; pub use model_ids::{ is_legacy_tier_model, legacy_tier_role, DEFAULT_MEMORY_SYNC_INTERVAL_SECS, DEFAULT_MODEL, - LEGACY_TIER_MODELS, MEMORY_SYNC_INTERVAL_PRESETS_SECS, MODEL_MANAGED_DEFAULT, WORKLOAD_ROLES, + LEGACY_TIER_MODELS, MANAGED_MULTIMODAL_MODELS, MEMORY_SYNC_INTERVAL_PRESETS_SECS, + MODEL_IMAGE_GENERATION_AGENT, MODEL_MANAGED_DEFAULT, MODEL_MEDIA_UNDERSTANDING, + MODEL_VIDEO_GENERATION_AGENT, WORKLOAD_ROLES, }; pub use output_language::{normalize_output_language, output_language_directive}; diff --git a/crates/openhuman-core/src/config/schema/types/model_ids.rs b/crates/openhuman-core/src/config/schema/types/model_ids.rs index 1588e4338b..2153533ae8 100644 --- a/crates/openhuman-core/src/config/schema/types/model_ids.rs +++ b/crates/openhuman-core/src/config/schema/types/model_ids.rs @@ -64,6 +64,42 @@ pub const WORKLOAD_ROLES: [&str; 8] = [ "subconscious", ]; +/// `hint:vision` is deprecated: `vision-v1` silently falls back to the chat +/// default on managed routes, which means every agent still pinned to it +/// loses image/video understanding without any error surfacing (regression +/// R4). Image/video UNDERSTANDING and GENERATION are also separate +/// capabilities that a single `vision` hint cannot distinguish, so each media +/// agent is pinned to its own exact OpenRouter passthrough model instead. +/// +/// Qwen3.7 Flash: cheap, native tool calling, text+image+video input, 1M +/// context, $0.03 / $0.13 per 1M input/output tokens. +/// +/// Used by `vision_agent` (image/video understanding: describe, OCR, chart +/// and UI-element reading). +pub const MODEL_MEDIA_UNDERSTANDING: &str = "openrouter/qwen/qwen3.7-flash"; + +/// Same model as [`MODEL_MEDIA_UNDERSTANDING`], pinned separately for +/// `image_agent` (image GENERATION delegate) so the two roles can be retuned +/// independently without one edit silently moving the other. +pub const MODEL_IMAGE_GENERATION_AGENT: &str = "openrouter/qwen/qwen3.7-flash"; + +/// Same model as [`MODEL_MEDIA_UNDERSTANDING`], pinned separately for +/// `video_agent` (video GENERATION delegate) so the two roles can be retuned +/// independently without one edit silently moving the other. +pub const MODEL_VIDEO_GENERATION_AGENT: &str = "openrouter/qwen/qwen3.7-flash"; + +/// Every managed model id that carries multimodal (image/video) input +/// capability, whether or not it is also the workload's `vision` hint +/// target. `oh_tier_supports_vision` treats membership here the same as the +/// legacy `vision-v1` / `hint:vision` gate, so a media agent pinned to one of +/// these `exact` ids keeps the image/video forwarding path that used to key +/// off the retired hint alone. +pub const MANAGED_MULTIMODAL_MODELS: [&str; 3] = [ + MODEL_MEDIA_UNDERSTANDING, + MODEL_IMAGE_GENERATION_AGENT, + MODEL_VIDEO_GENERATION_AGENT, +]; + /// Effective default global memory-sync cadence (seconds) used when /// [`Config::memory_sync_interval_secs`] is `None` — i.e. the user has not /// explicitly picked a schedule. 24h, matching the "Sync every 24h" preset diff --git a/crates/openhuman-core/src/inference/model_context_tests.rs b/crates/openhuman-core/src/inference/model_context_tests.rs index 1ca097c8cb..571518e907 100644 --- a/crates/openhuman-core/src/inference/model_context_tests.rs +++ b/crates/openhuman-core/src/inference/model_context_tests.rs @@ -141,6 +141,12 @@ fn oh_tier_vision_map_is_exhaustively_pinned() { "hint:reasoning", "vision-v1", "hint:vision", + // The dedicated OpenRouter passthrough models the media agents + // (`vision_agent`, `image_agent`, `video_agent`) are pinned to now + // that `hint:vision` / `vision-v1` is deprecated (regression R4). + crate::config::MODEL_MEDIA_UNDERSTANDING, + crate::config::MODEL_IMAGE_GENERATION_AGENT, + crate::config::MODEL_VIDEO_GENERATION_AGENT, ] { assert!( oh_tier_supports_vision(tier), @@ -170,3 +176,25 @@ fn oh_tier_vision_map_is_exhaustively_pinned() { ); } } + +/// R4 regression: the media agents' new `exact` model pin +/// (`openrouter/qwen/qwen3.7-flash`, replacing the deprecated `hint:vision`) +/// must be reported vision-capable through both the tier-map gate and the +/// combined `model_supports_vision` facade — the two call sites +/// `dispatch.rs:256` and `runner.rs:1536` actually gate image/video +/// forwarding on. +#[test] +fn media_agent_pinned_model_is_vision_capable() { + use crate::config::{Config, MODEL_MEDIA_UNDERSTANDING}; + use crate::inference::provider::factory::oh_tier_supports_vision; + + assert!( + oh_tier_supports_vision(MODEL_MEDIA_UNDERSTANDING), + "{MODEL_MEDIA_UNDERSTANDING} must be reported vision-capable" + ); + let config = Config::default(); + assert!( + model_supports_vision(MODEL_MEDIA_UNDERSTANDING, &config), + "{MODEL_MEDIA_UNDERSTANDING} must be vision-capable through the combined facade too" + ); +} diff --git a/crates/openhuman-core/src/inference/provider/factory/tiers.rs b/crates/openhuman-core/src/inference/provider/factory/tiers.rs index fed2debed7..9941c4221e 100644 --- a/crates/openhuman-core/src/inference/provider/factory/tiers.rs +++ b/crates/openhuman-core/src/inference/provider/factory/tiers.rs @@ -9,7 +9,7 @@ //! as aliases of their role so un-migrated callers keep routing. use super::*; -use crate::config::{legacy_tier_role, MODEL_MANAGED_DEFAULT}; +use crate::config::{legacy_tier_role, MANAGED_MULTIMODAL_MODELS, MODEL_MANAGED_DEFAULT}; /// Whether `model` is a managed alias rather than a concrete model id: a /// `hint:*` role marker or a retired tier slug. @@ -134,12 +134,18 @@ pub(crate) fn is_raw_passthrough_model(model: &str) -> bool { /// The managed backend does not advertise per-model capabilities, so the core /// owns this. [`MODEL_MANAGED_DEFAULT`] (DeepSeek V4 Flash on the managed /// backend) accepts images, as do the `vision` and `reasoning` role aliases -/// (and their retired tier slugs) that always ran a multimodal model. Any other -/// pinned catalog id is covered by the user's `model_registry.vision` flag +/// (and their retired tier slugs) that always ran a multimodal model, and +/// every exact id in [`MANAGED_MULTIMODAL_MODELS`] — the OpenRouter +/// passthrough models the media agents (`vision_agent`, `image_agent`, +/// `video_agent`) are pinned to now that `hint:vision` / `vision-v1` is +/// deprecated (regression R4: the retired hint silently fell back to the +/// chat default on managed routes, so an agent still pinned to it lost image +/// forwarding with no error). Any other pinned catalog id is covered by the +/// user's `model_registry.vision` flag /// ([`crate::inference::model_context::model_vision_enabled`]). pub(crate) fn oh_tier_supports_vision(model: &str) -> bool { let trimmed = model.trim(); - if trimmed == MODEL_MANAGED_DEFAULT { + if trimmed == MODEL_MANAGED_DEFAULT || MANAGED_MULTIMODAL_MODELS.contains(&trimmed) { return true; } matches!( diff --git a/crates/openhuman-core/src/media/generation/download.rs b/crates/openhuman-core/src/media/generation/download.rs deleted file mode 100644 index c7ca9e1165..0000000000 --- a/crates/openhuman-core/src/media/generation/download.rs +++ /dev/null @@ -1,149 +0,0 @@ -//! Persist generated media to the agent's action directory. -//! -//! GMI returns expiring signed URLs; we download the bytes and write them under -//! a `generated-media/` root inside `action_dir` so final answers can reference -//! a stable local file path (per the `image_generation` contract). The action -//! directory is the agent's canonical read/write root. - -use std::path::{Path, PathBuf}; - -use anyhow::{Context, Result}; - -use super::types::MediaItem; - -/// Subdirectory (under `action_dir`) where generated artifacts are stored. -const GENERATED_MEDIA_DIR: &str = "generated-media"; - -/// A downloaded artifact and where it landed on disk. -#[derive(Debug, Clone)] -pub struct PersistedArtifact { - pub kind: String, - pub path: PathBuf, - pub source_url: String, - pub thumbnail_url: Option, -} - -/// Pick a file extension from the artifact kind + content type / URL. -fn extension_for(kind: &str, content_type: Option<&str>, url: &str) -> String { - if let Some(ct) = content_type { - let ct = ct.to_ascii_lowercase(); - if ct.contains("png") { - return "png".to_string(); - } - if ct.contains("webp") { - return "webp".to_string(); - } - if ct.contains("jpeg") || ct.contains("jpg") { - return "jpg".to_string(); - } - if ct.contains("mp4") { - return "mp4".to_string(); - } - if ct.contains("webm") { - return "webm".to_string(); - } - } - // Fall back to the URL path suffix, then a per-kind default. - let lower = url.split('?').next().unwrap_or(url).to_ascii_lowercase(); - for ext in ["png", "webp", "jpg", "jpeg", "mp4", "webm"] { - if lower.ends_with(&format!(".{ext}")) { - return if ext == "jpeg" { - "jpg".to_string() - } else { - ext.to_string() - }; - } - } - if kind.eq_ignore_ascii_case("video") { - "mp4".to_string() - } else { - "png".to_string() - } -} - -/// Download a single media URL into `dir`, returning the written path. -async fn download_one( - http: &reqwest::Client, - dir: &Path, - item: &MediaItem, - request_id: &str, - index: usize, -) -> Result { - tracing::info!( - "[media_generation] downloading {} artifact {} for request={}", - item.kind, - index, - request_id - ); - let resp = http - .get(&item.url) - .send() - .await - .with_context(|| format!("failed to fetch generated media from {}", item.url))? - .error_for_status() - .with_context(|| format!("generated media URL returned an error: {}", item.url))?; - - let content_type = resp - .headers() - .get(reqwest::header::CONTENT_TYPE) - .and_then(|v| v.to_str().ok()) - .map(|s| s.to_string()); - let ext = extension_for(&item.kind, content_type.as_deref(), &item.url); - - let bytes = resp - .bytes() - .await - .with_context(|| format!("failed to read generated media body from {}", item.url))?; - - // Sanitize the request id for use in a filename (it is a UUID from GMI, but - // be defensive against path separators). - let safe_id: String = request_id - .chars() - .map(|c| { - if c.is_ascii_alphanumeric() || c == '-' { - c - } else { - '_' - } - }) - .collect(); - let filename = format!("{safe_id}-{index}.{ext}"); - let path = dir.join(&filename); - tokio::fs::write(&path, &bytes) - .await - .with_context(|| format!("failed to write generated media to {}", path.display()))?; - - Ok(PersistedArtifact { - kind: item.kind.clone(), - path, - source_url: item.url.clone(), - thumbnail_url: item.thumbnail_url.clone(), - }) -} - -/// Download + persist all media items for a request under -/// `{action_dir}/generated-media/`. Returns the written artifacts. -pub async fn persist_media( - action_dir: &Path, - request_id: &str, - items: &[MediaItem], -) -> Result> { - if items.is_empty() { - return Ok(Vec::new()); - } - let dir = action_dir.join(GENERATED_MEDIA_DIR); - tokio::fs::create_dir_all(&dir) - .await - .with_context(|| format!("failed to create generated-media dir {}", dir.display()))?; - - let http = reqwest::Client::new(); - let mut out = Vec::with_capacity(items.len()); - for (i, item) in items.iter().enumerate() { - out.push(download_one(&http, &dir, item, request_id, i).await?); - } - Ok(out) -} - -#[cfg(test)] -#[path = "download_tests.rs"] -mod tests; diff --git a/crates/openhuman-core/src/media/generation/download_tests.rs b/crates/openhuman-core/src/media/generation/download_tests.rs deleted file mode 100644 index b1f2539f3d..0000000000 --- a/crates/openhuman-core/src/media/generation/download_tests.rs +++ /dev/null @@ -1,27 +0,0 @@ -use super::*; - -#[test] -fn extension_prefers_content_type() { - assert_eq!( - extension_for("image", Some("image/png"), "https://x/y"), - "png" - ); - assert_eq!( - extension_for("image", Some("image/webp"), "https://x/y"), - "webp" - ); - assert_eq!( - extension_for("video", Some("video/mp4"), "https://x/y"), - "mp4" - ); -} - -#[test] -fn extension_falls_back_to_url_then_kind() { - assert_eq!( - extension_for("image", None, "https://x/y/a.webp?sig=1"), - "webp" - ); - assert_eq!(extension_for("video", None, "https://x/y/clip"), "mp4"); - assert_eq!(extension_for("image", None, "https://x/y/clip"), "png"); -} diff --git a/crates/openhuman-core/src/media/generation/mod.rs b/crates/openhuman-core/src/media/generation/mod.rs index 2b0d699c35..d771ce074c 100644 --- a/crates/openhuman-core/src/media/generation/mod.rs +++ b/crates/openhuman-core/src/media/generation/mod.rs @@ -1,15 +1,24 @@ -//! Media generation domain — agent tools for image/video generation backed by -//! GMI via the OpenHuman backend's `media_generation` provider. +//! Media generation domain — image and video generation agent tools backed by +//! OpenRouter through the OpenHuman backend's `/agent-integrations/openrouter` +//! proxy. //! -//! The backend (`/agent-integrations/media-generation/*`) owns provider keys, -//! billing, and the standardized contract; these tools submit a request, block -//! with progress until it completes, download the resulting media into the -//! agent's `generated-media/` root, and return local file paths. +//! The work is split by ownership: +//! +//! - **TinyInference** (`tinyinference-image` / `tinyinference-video`, reached +//! through `tinyagents_harness`) owns the wire contract, reference and +//! output-shape standards, the submit → poll → download job loop, and the +//! rule that a billed call returns media or an error, never an empty success. +//! - **TinyAgents** (`tinyagents_harness::media`) owns the tools: argument +//! parsing, artifact persistence into the workspace, and result wording. +//! - **This module** owns the host policy: endpoint, credential, egress, +//! privacy and budget gates ([`provider`]), plus tool names, descriptions and +//! the local-reference policy ([`tools`]). -pub mod download; +pub mod provider; pub mod tools; -pub mod types; +pub use provider::{managed_generators, MediaGenerators, OPENROUTER_PROXY_PATH}; pub use tools::{ - build_media_tools, MediaGenerateImageTool, MediaGenerateVideoTool, MediaListModelsTool, + build_media_tools, media_tools_from, MediaListModelsTool, IMAGE_TOOL_NAME, + LIST_MODELS_TOOL_NAME, VIDEO_TOOL_NAME, }; diff --git a/crates/openhuman-core/src/media/generation/provider.rs b/crates/openhuman-core/src/media/generation/provider.rs new file mode 100644 index 0000000000..4a254e765d --- /dev/null +++ b/crates/openhuman-core/src/media/generation/provider.rs @@ -0,0 +1,199 @@ +//! Host adapter: OpenRouter media generators reached through the OpenHuman +//! backend's `/agent-integrations/openrouter` proxy. +//! +//! TinyInference owns the wire contract (`POST /images`, `POST /videos`, +//! `GET /videos/{id}`, `GET /videos/{id}/content`); this module supplies only +//! what the host owns: +//! +//! - **Endpoint and headers** — the backend transport's `raw_client()` (which +//! already carries `x-sdk-name` and the version headers) and the +//! `/agent-integrations/openrouter` base URL. +//! - **Credential** — a per-request resolver over +//! [`resolve_backend_credential`], so a desktop session JWT and a library +//! API key both work and a refreshed session is picked up. +//! - **Policy** — local-only enforcement and the egress disclosure before any +//! request leaves the device, and the managed-credit budget gate before a +//! billed submit. These are the same gates `IntegrationClient` applies to +//! every `/agent-integrations/*` call. + +use std::sync::Arc; + +use async_trait::async_trait; +use tinyagents_harness::tinyinference_image::{ + self as ti_image, BearerResolver, GeneratedMedia, ImageGenerator, ImageRequest, ImageResponse, + MediaAuth, MediaModel, MediaTransport, OpenRouterImageGenerator, +}; +use tinyagents_harness::tinyinference_video::{ + self as ti_video, OpenRouterVideoGenerator, VideoGenerator, VideoJob, VideoJobStatus, + VideoRequest, +}; + +use crate::api::config::effective_backend_api_url; +use crate::api::BackendOAuthClient; +use crate::config::Config; +use crate::security::credentials::session_support::{ + resolve_backend_credential, BackendCredential, +}; + +/// Backend route prefix that proxies OpenRouter's media API. +pub const OPENROUTER_PROXY_PATH: &str = "/agent-integrations/openrouter"; + +/// The image and video generators for this process. +pub struct MediaGenerators { + /// Image generation. + pub image: Arc, + /// Video generation. + pub video: Arc, +} + +/// Builds generators against the managed backend, or `None` when no backend +/// transport is installed (a core with no TinyHumans connection). +pub fn managed_generators(config: &Config) -> Option { + let client = match BackendOAuthClient::new(&effective_backend_api_url(&config.api_url)) { + Ok(client) => client, + Err(error) => { + tracing::debug!(%error, "[media_generation] invalid backend URL; media tools skipped"); + return None; + } + }; + let http = match client.raw_client() { + Ok(http) => http, + Err(error) => { + tracing::debug!(%error, "[media_generation] no backend transport; media tools skipped"); + return None; + } + }; + let base = match client.url_for(OPENROUTER_PROXY_PATH) { + Ok(url) => url, + Err(error) => { + tracing::debug!(%error, "[media_generation] cannot build proxy URL; media tools skipped"); + return None; + } + }; + let config = Arc::new(config.clone()); + let transport = MediaTransport::new(MediaAuth::Bearer(bearer_resolver(Arc::clone(&config)))) + .with_client(http) + .with_base_url(base.as_str()); + tracing::debug!(base = %base, "[media_generation] managed OpenRouter media generators ready"); + Some(MediaGenerators { + image: Arc::new(GuardedImage { + inner: OpenRouterImageGenerator::with_transport(transport.clone()), + guard: Guard { + config: Arc::clone(&config), + }, + }), + video: Arc::new(GuardedVideo { + inner: OpenRouterVideoGenerator::with_transport(transport), + guard: Guard { config }, + }), + }) +} + +/// Resolves the backend credential on every request, so a session refreshed +/// mid-run is used and an API-key host needs no session at all. +pub(crate) fn bearer_resolver(config: Arc) -> BearerResolver { + Arc::new(move || match resolve_backend_credential(&config) { + Ok(BackendCredential::Session(token) | BackendCredential::ApiKey(token)) => Ok(token), + Err(error) => Err(ti_image::Error::Auth(error)), + }) +} + +/// Host policy applied before a request leaves the device. +struct Guard { + config: Arc, +} + +impl Guard { + /// Local-only enforcement, then the egress disclosure. A blocked call is + /// neither disclosed nor sent. + fn admit(&self, route: &str) -> ti_image::Result<()> { + let descriptor = crate::security::egress::EgressDescriptor::integration(format!( + "{OPENROUTER_PROXY_PATH}/{route}" + )); + crate::security::egress::enforce_egress(&descriptor).map_err(|error| { + tracing::info!(route, %error, "[media_generation] blocked by privacy policy"); + ti_image::Error::Validation(format!("blocked by the privacy policy: {error}")) + })?; + crate::security::egress::emit_external_transfer(descriptor); + Ok(()) + } + + /// Refuses a billed submit when managed credits are exhausted. + async fn budget(&self) -> ti_image::Result<()> { + if crate::integrations::client::budget_gate::managed_tool_budget_exhausted(&self.config) + .await + { + tracing::info!("[media_generation] managed credits exhausted; submit refused"); + return Err(ti_image::Error::Validation( + "Managed cloud tools are disabled because your OpenHuman AI credits are exhausted. \ + Add credits or route the task to user-supplied providers." + .into(), + )); + } + Ok(()) + } +} + +struct GuardedImage { + inner: OpenRouterImageGenerator, + guard: Guard, +} + +#[async_trait] +impl ImageGenerator for GuardedImage { + fn name(&self) -> &str { + "openhuman-openrouter" + } + + fn default_model(&self) -> &str { + self.inner.default_model() + } + + async fn generate(&self, request: ImageRequest) -> ti_image::Result { + self.guard.admit("images")?; + self.guard.budget().await?; + self.inner.generate(request).await + } + + async fn list_models(&self) -> ti_image::Result> { + self.guard.admit("images/models")?; + self.inner.list_models().await + } +} + +struct GuardedVideo { + inner: OpenRouterVideoGenerator, + guard: Guard, +} + +#[async_trait] +impl VideoGenerator for GuardedVideo { + fn name(&self) -> &str { + "openhuman-openrouter" + } + + fn default_model(&self) -> &str { + self.inner.default_model() + } + + async fn submit(&self, request: VideoRequest) -> ti_video::Result { + self.guard.admit("videos")?; + self.guard.budget().await?; + self.inner.submit(request).await + } + + async fn poll(&self, job_id: &str) -> ti_video::Result { + self.guard.admit("videos/{jobId}")?; + self.inner.poll(job_id).await + } + + async fn content(&self, job_id: &str, index: usize) -> ti_video::Result { + self.guard.admit("videos/{jobId}/content")?; + self.inner.content(job_id, index).await + } + + async fn list_models(&self) -> ti_video::Result> { + self.guard.admit("videos/models")?; + self.inner.list_models().await + } +} diff --git a/crates/openhuman-core/src/media/generation/tools.rs b/crates/openhuman-core/src/media/generation/tools.rs index 192c25dcca..33eaa6da5e 100644 --- a/crates/openhuman-core/src/media/generation/tools.rs +++ b/crates/openhuman-core/src/media/generation/tools.rs @@ -1,439 +1,150 @@ -//! Agent-facing media-generation tools (image + video) backed by GMI via the -//! OpenHuman backend's `media_generation` provider. +//! Media generation agent tools. //! -//! **Endpoints** (see `backend/docs/media-generation.md`): -//! - `POST /agent-integrations/media-generation/images` -//! - `POST /agent-integrations/media-generation/videos` -//! - `GET /agent-integrations/media-generation/requests/{requestId}` -//! - `GET /agent-integrations/media-generation/models` -//! -//! Generation is asynchronous. These tools **block with progress**: they submit -//! (`wait:false`, so the backend charges + returns a request id immediately), -//! then poll the request until it reaches a terminal state, download each -//! resulting artifact into the agent's `generated-media/` root, and return the -//! local file paths. If the request does not reach a terminal state within the -//! wait budget, the tool returns an error (never a success with no file). The -//! backend owns GMI keys, billing, and rate limiting. +//! `media_generate_image` and `media_generate_video` are TinyAgents' +//! [`GenerateImageTool`] / [`GenerateVideoTool`] bound to the managed +//! generators from [`super::provider`], under the names the `media` tool pack +//! and the image/video agents' allowlists already use. `media_list_models` +//! lists what the generators can run. -use std::path::{Path, PathBuf}; +use std::path::{Component, Path, PathBuf}; use std::sync::Arc; -use std::time::{Duration, Instant}; +use std::time::Duration; use async_trait::async_trait; use serde_json::{json, Value}; +use tinyagents_harness::media::{GenerateImageTool, GenerateVideoTool, MediaOutput}; +use tinyagents_harness::tinyinference_image::ImageGenerator; +use tinyagents_harness::tinyinference_video::{VideoGenerator, WaitPolicy}; +use tinytools::{PermissionLevel, Tool, ToolCategory, ToolResult}; +use super::provider::{managed_generators, MediaGenerators}; use crate::config::Config; -use crate::integrations::IntegrationClient; -use tinytools::ToolRunContext; -use tinytools::{PermissionLevel, Tool, ToolCallOptions, ToolCategory, ToolResult}; - -use super::download::persist_media; -use super::types::MediaResponse; - -const IMAGES_PATH: &str = "/agent-integrations/media-generation/images"; -const VIDEOS_PATH: &str = "/agent-integrations/media-generation/videos"; -const MODELS_PATH: &str = "/agent-integrations/media-generation/models"; -/// Poll cadence + caps. Images are fast; video can take minutes. -const POLL_INTERVAL: Duration = Duration::from_secs(4); -const IMAGE_MAX_WAIT_SECS: u64 = 300; -const VIDEO_MAX_WAIT_SECS: u64 = 420; - -/// Shared submit-then-poll-then-persist flow for both modalities. -async fn generate_and_persist( - client: &IntegrationClient, - action_dir: &Path, - submit_path: &str, - body: Value, - max_wait_secs: u64, -) -> ToolResult { - // Submit without server-side blocking; the backend charges on submit and - // returns a request id we poll ourselves (so the core owns the progress UX). - let submitted: MediaResponse = match client.post::(submit_path, &body).await { - Ok(resp) => resp, - Err(e) => return ToolResult::error(format!("Media generation submit failed: {e}")), +/// Image tool name (pinned by the `media` pack and agent allowlists). +pub const IMAGE_TOOL_NAME: &str = "media_generate_image"; +/// Video tool name. +pub const VIDEO_TOOL_NAME: &str = "media_generate_video"; +/// Model-listing tool name. +pub const LIST_MODELS_TOOL_NAME: &str = "media_list_models"; + +/// Poll cadence and budget for a video job. +const VIDEO_POLL_INTERVAL: Duration = Duration::from_secs(5); +const VIDEO_WAIT_BUDGET: Duration = Duration::from_secs(600); + +const IMAGE_DESCRIPTION: &str = "Generate or edit images from a text prompt via OpenRouter \ + (default model: Seedream 5.0 Lite). Pass `references` (https URLs or workspace file paths) \ + to edit, restyle, or keep a subject consistent. Saves each image under the workspace \ + `generated-media/` folder and returns the file path. Billed per call: after an error that \ + says the call was billed, do not call again — report it to the user."; + +const VIDEO_DESCRIPTION: &str = "Generate a short video clip via OpenRouter (default model: \ + Seedance 2.0 Mini, 4–15 s, 480p/720p, optional audio). Optionally start from \ + `first_frame` or end on `last_frame` (URL or workspace path). Blocks until the clip is \ + ready (minutes) and saves it under `generated-media/`. Billed per call: if it times out, \ + call again with `resume_job_id` instead of submitting a new job."; + +/// Registers the media tools, or nothing when no backend is reachable. +pub fn build_media_tools(root_config: &Config, action_dir: &Path) -> Vec> { + let Some(generators) = managed_generators(root_config) else { + return Vec::new(); }; - - let request_id = submitted.request_id.clone(); - tracing::info!( - "[media_generation] submitted request={} status={} cost=${:.4}", - request_id, - submitted.status, - submitted.cost_usd - ); - - let status_path = format!( - "/agent-integrations/media-generation/requests/{}", - request_id - ); - - let mut latest = submitted; - let mut last_poll_error: Option = None; - let deadline = Instant::now() + Duration::from_secs(max_wait_secs); - while !latest.is_terminal() { - if Instant::now() >= deadline { - tracing::warn!( - "[media_generation] wait budget elapsed for request={} (status={}, last_poll_error={:?})", - request_id, - latest.status, - last_poll_error - ); - // The generation never reached a terminal state within the budget, so - // no artifact was downloaded. Surface an error rather than a false - // success — a caller told "success" would report a file that was never - // produced. Keep the message stable and free of upstream error text - // (that stays in the log line above): the request was accepted and - // billed and may still be running server-side, and there is no - // resume-by-id path, so retrying submits and bills a brand-new - // generation. - return ToolResult::error(format!( - "Media generation did not complete within {max_wait_secs}s (request_id: \ - {request_id}, last status: {}). The request was accepted and billed and may \ - still be running on the server. Calling this tool again starts and bills a \ - separate generation (there is no resume-by-id), so do not retry automatically — \ - report this to the user and let them decide.", - latest.status - )); - } - // Don't sleep past the deadline: cap the poll interval to the time left so - // the wait budget is enforced before each poll. The poll request itself is - // bounded by the integration client's request timeout. - let remaining = deadline.saturating_duration_since(Instant::now()); - tokio::time::sleep(POLL_INTERVAL.min(remaining)).await; - match client.get::(&status_path).await { - Ok(resp) => { - tracing::debug!( - "[media_generation] poll request={} status={}", - request_id, - resp.status - ); - latest = resp; - } - Err(e) => { - tracing::warn!( - "[media_generation] poll error for request={}: {e}", - request_id - ); - // Transient poll failures shouldn't abort a paid generation — keep - // polling until the deadline, but remember the last error so a - // timeout can explain why it never observed a terminal status. - last_poll_error = Some(e.to_string()); - } - } - } - - if latest.is_failed() { - return ToolResult::error(format!( - "Media generation failed (request_id: {request_id})." - )); - } - - if latest.media.is_empty() { - return ToolResult::error(format!( - "Media generation reported success but returned no media (request_id: {request_id})." - )); - } - - match persist_media(action_dir, &request_id, &latest.media).await { - Ok(artifacts) => { - let mut lines = vec![format!( - "Generated {} artifact(s) (request_id: {}, model: {}):", - artifacts.len(), - request_id, - latest.model - )]; - for art in &artifacts { - lines.push(format!("- {} → {}", art.kind, art.path.display())); - if let Some(thumb) = &art.thumbnail_url { - lines.push(format!(" thumbnail: {thumb}")); - } - } - lines.push(format!("\nCost: ${:.4}", latest.cost_usd)); - let payload = json!({ - "request_id": request_id, - "model": latest.model, - "cost_usd": latest.cost_usd, - "artifacts": artifacts.iter().map(|a| json!({ - "type": a.kind, - "path": a.path.display().to_string(), - "source_url": a.source_url, - "thumbnail_url": a.thumbnail_url, - })).collect::>(), - }); - ToolResult::success_with_markdown(payload, lines.join("\n")) - } - Err(e) => ToolResult::error(format!( - "Generation succeeded but persisting media failed (request_id: {request_id}): {e}" - )), - } -} - -fn action_dir_for_context( - default_action_dir: &Path, - context: Option<&dyn ToolRunContext>, - tool_name: &str, -) -> PathBuf { - if let Some(workspace) = context.and_then(|ctx| ctx.workspace()) { - tracing::debug!( - tool = tool_name, - workspace_root = %workspace.root.display(), - policy_id = %workspace.policy_id, - "[media_generation] using ToolExecutionContext workspace root" - ); - return workspace.root.clone(); - } - - default_action_dir.to_path_buf() -} - -// ── MediaGenerateImageTool ────────────────────────────────────────── - -pub struct MediaGenerateImageTool { - client: Arc, - action_dir: PathBuf, -} - -impl MediaGenerateImageTool { - pub fn new(client: Arc, action_dir: PathBuf) -> Self { - Self { client, action_dir } - } - - async fn run(&self, args: Value, action_dir: &Path) -> anyhow::Result { - let prompt = match args.get("prompt").and_then(|v| v.as_str()) { - Some(p) if !p.trim().is_empty() => p, - _ => return Ok(ToolResult::error("prompt is required")), - }; - - let mut body = json!({ "prompt": prompt, "wait": false }); - if let Some(model) = args.get("model").and_then(|v| v.as_str()) { - body["model"] = json!(model); - } - if let Some(size) = args.get("size").and_then(|v| v.as_str()) { - body["size"] = json!(size); - } - if let Some(n) = args.get("n").and_then(|v| v.as_u64()) { - body["n"] = json!(n.clamp(1, 8)); - } - if let Some(imgs) = args.get("input_images").and_then(|v| v.as_array()) { - let urls: Vec<&str> = imgs.iter().filter_map(|v| v.as_str()).collect(); - if !urls.is_empty() { - body["inputImages"] = json!(urls); - } - } - if let Some(seed) = args.get("seed").and_then(|v| v.as_i64()) { - body["seed"] = json!(seed); - } - - tracing::info!( - prompt_len = prompt.len(), - action_dir = %action_dir.display(), - "[media_generate_image] persisting generated media" - ); - Ok(generate_and_persist( - &self.client, - action_dir, - IMAGES_PATH, - body, - IMAGE_MAX_WAIT_SECS, - ) - .await) - } -} - -#[async_trait] -impl Tool for MediaGenerateImageTool { - fn name(&self) -> &str { - "media_generate_image" - } - - fn description(&self) -> &str { - "Generate or edit an image from a text prompt using GMI (Seedream / SeedEdit). \ - Optionally pass reference image URLs to edit/condition (image-to-image). \ - Blocks until the image is ready and saves it under the workspace \ - generated-media folder, returning the local file path. Cost is billed by the backend." - } - - fn parameters_schema(&self) -> Value { - json!({ - "type": "object", - "properties": { - "prompt": { "type": "string", "description": "Detailed visual prompt or edit instruction" }, - "model": { "type": "string", "description": "Optional GMI model id (default: seedream-4-0-250828). Use media_list_models to discover." }, - "size": { "type": "string", "description": "Optional output size, e.g. 1024x1024 or 1536x1024" }, - "n": { "type": "integer", "minimum": 1, "maximum": 8, "description": "Number of images (default 1)" }, - "input_images": { - "type": "array", - "items": { "type": "string" }, - "description": "Optional reference image URLs for edit / image-to-image" - }, - "seed": { "type": "integer", "description": "Optional seed for reproducibility" } - }, - "required": ["prompt"] - }) - } - - fn permission_level(&self) -> PermissionLevel { - PermissionLevel::Execute - } - - fn category(&self) -> ToolCategory { - ToolCategory::Workflow - } - - async fn execute(&self, args: Value) -> anyhow::Result { - self.run(args, &self.action_dir).await - } - - async fn execute_with_context( - &self, - args: Value, - _options: ToolCallOptions, - context: Option<&dyn ToolRunContext>, - ) -> anyhow::Result { - let action_dir = action_dir_for_context(&self.action_dir, context, self.name()); - self.run(args, &action_dir).await - } + media_tools_from( + generators, + action_dir, + &root_config.workspace_dir, + WaitPolicy::new(VIDEO_POLL_INTERVAL, VIDEO_WAIT_BUDGET), + ) } -// ── MediaGenerateVideoTool ────────────────────────────────────────── - -pub struct MediaGenerateVideoTool { - client: Arc, - action_dir: PathBuf, +/// Builds the tool set over any generators (the managed ones in production, +/// mocks in tests), writing under `action_dir`; `video_wait` bounds each +/// video job. +pub fn media_tools_from( + generators: MediaGenerators, + action_dir: &Path, + workspace_dir: &Path, + video_wait: WaitPolicy, +) -> Vec> { + let MediaGenerators { image, video } = generators; + let output = MediaOutput::new(action_dir) + .with_reference_policy(reference_policy(action_dir, workspace_dir)); + let tools: Vec> = vec![ + Box::new( + GenerateImageTool::new(Arc::clone(&image), output.clone()) + .with_name(IMAGE_TOOL_NAME) + .with_description(IMAGE_DESCRIPTION) + .with_permission_level(PermissionLevel::Execute) + .with_category(ToolCategory::Workflow), + ), + Box::new( + GenerateVideoTool::new(Arc::clone(&video), output) + .with_name(VIDEO_TOOL_NAME) + .with_description(VIDEO_DESCRIPTION) + .with_permission_level(PermissionLevel::Execute) + .with_category(ToolCategory::Workflow) + .with_wait_policy(video_wait), + ), + Box::new(MediaListModelsTool { image, video }), + ]; + tracing::debug!("[media_generation] registered {} media tools", tools.len()); + tools } -impl MediaGenerateVideoTool { - pub fn new(client: Arc, action_dir: PathBuf) -> Self { - Self { client, action_dir } - } - - async fn run(&self, args: Value, action_dir: &Path) -> anyhow::Result { - let prompt = match args.get("prompt").and_then(|v| v.as_str()) { - Some(p) if !p.trim().is_empty() => p, - _ => return Ok(ToolResult::error("prompt is required")), - }; - - let mut body = json!({ "prompt": prompt, "wait": false }); - if let Some(model) = args.get("model").and_then(|v| v.as_str()) { - body["model"] = json!(model); - } - if let Some(img) = args.get("input_image").and_then(|v| v.as_str()) { - body["inputImage"] = json!(img); - } - if let Some(d) = args.get("duration_seconds").and_then(|v| v.as_u64()) { - body["durationSeconds"] = json!(d.clamp(1, 60)); - } - if let Some(ar) = args.get("aspect_ratio").and_then(|v| v.as_str()) { - body["aspectRatio"] = json!(ar); +/// Local reference files may be read and uploaded only from the action +/// directory or the workspace directory, never through `..`, and never from +/// an always-forbidden location (credential stores, system roots). +pub(crate) fn reference_policy( + action_dir: &Path, + workspace_dir: &Path, +) -> tinyagents_harness::media::ReferencePathPolicy { + let roots: Vec = vec![action_dir.to_path_buf(), workspace_dir.to_path_buf()]; + Arc::new(move |path: &Path| { + if path.components().any(|c| matches!(c, Component::ParentDir)) { + return Err(format!( + "reference path {} may not contain '..'", + path.display() + )); } - if let Some(np) = args.get("negative_prompt").and_then(|v| v.as_str()) { - body["negativePrompt"] = json!(np); + if crate::security::SecurityPolicy::is_always_forbidden(path) { + return Err(format!( + "reference path {} is in a protected location", + path.display() + )); } - if let Some(seed) = args.get("seed").and_then(|v| v.as_i64()) { - body["seed"] = json!(seed); + if !roots.iter().any(|root| path.starts_with(root)) { + return Err(format!( + "reference path {} is outside the workspace; use a URL or a file inside the workspace", + path.display() + )); } - - tracing::info!( - prompt_len = prompt.len(), - action_dir = %action_dir.display(), - "[media_generate_video] persisting generated media" - ); - Ok(generate_and_persist( - &self.client, - action_dir, - VIDEOS_PATH, - body, - VIDEO_MAX_WAIT_SECS, - ) - .await) - } + Ok(path.to_path_buf()) + }) } -#[async_trait] -impl Tool for MediaGenerateVideoTool { - fn name(&self) -> &str { - "media_generate_video" - } - - fn description(&self) -> &str { - "Generate a short video from a text prompt using GMI (Seedance / Veo). \ - Optionally pass a first-frame/reference image URL for image-to-video. \ - Video can take a few minutes; this blocks until it is ready, saves the \ - clip under the workspace generated-media folder, and returns the local \ - file path. Cost is billed by the backend." - } - - fn parameters_schema(&self) -> Value { - json!({ - "type": "object", - "properties": { - "prompt": { "type": "string", "description": "Detailed description of the video to generate" }, - "model": { "type": "string", "description": "Optional GMI model id (default: seedance-1-0-pro-fast-251015). Use media_list_models to discover." }, - "input_image": { "type": "string", "description": "Optional first-frame / reference image URL for image-to-video" }, - "duration_seconds": { "type": "integer", "minimum": 1, "maximum": 60, "description": "Optional clip duration in seconds" }, - "aspect_ratio": { "type": "string", "description": "Optional aspect ratio, e.g. 16:9, 9:16, 1:1" }, - "negative_prompt": { "type": "string", "description": "Optional description of what to avoid" }, - "seed": { "type": "integer", "description": "Optional seed for reproducibility" } - }, - "required": ["prompt"] - }) - } - - fn permission_level(&self) -> PermissionLevel { - PermissionLevel::Execute - } - - fn category(&self) -> ToolCategory { - ToolCategory::Workflow - } - - async fn execute(&self, args: Value) -> anyhow::Result { - self.run(args, &self.action_dir).await - } - - async fn execute_with_context( - &self, - args: Value, - _options: ToolCallOptions, - context: Option<&dyn ToolRunContext>, - ) -> anyhow::Result { - let action_dir = action_dir_for_context(&self.action_dir, context, self.name()); - self.run(args, &action_dir).await - } -} - -// ── MediaListModelsTool ───────────────────────────────────────────── - +/// Lists the image and video models the generators can run. pub struct MediaListModelsTool { - client: Arc, -} - -impl MediaListModelsTool { - pub fn new(client: Arc) -> Self { - Self { client } - } + image: Arc, + video: Arc, } #[async_trait] impl Tool for MediaListModelsTool { fn name(&self) -> &str { - "media_list_models" + LIST_MODELS_TOOL_NAME } fn description(&self) -> &str { - "List available image/video generation models — a curated catalog with \ - pricing, plus (with include_upstream) GMI's full live model list. Use to \ - pick a `model` id for media_generate_image / media_generate_video." + "List the image and video generation models available to media_generate_image and \ + media_generate_video, with their ids. Use only when the user asks for a specific model \ + or style the default model cannot do." } fn parameters_schema(&self) -> Value { json!({ "type": "object", "properties": { - "include_upstream": { - "type": "boolean", - "description": "Also fetch GMI's full live model list (default false)" - } + "kind": { "type": "string", "enum": ["image", "video", "all"], "description": "Which catalog (default all)." }, + "search": { "type": "string", "description": "Case-insensitive substring filter on id or name." } } }) } @@ -442,52 +153,67 @@ impl Tool for MediaListModelsTool { ToolCategory::Workflow } + fn permission_level(&self) -> PermissionLevel { + PermissionLevel::ReadOnly + } + async fn execute(&self, args: Value) -> anyhow::Result { - let include_upstream = args - .get("include_upstream") - .and_then(|v| v.as_bool()) - .unwrap_or(false); - let path = if include_upstream { - format!("{MODELS_PATH}?includeUpstream=true") - } else { - MODELS_PATH.to_string() + let kind = args.get("kind").and_then(Value::as_str).unwrap_or("all"); + let search = args + .get("search") + .and_then(Value::as_str) + .map(str::to_ascii_lowercase); + let keep = |id: &str, name: Option<&str>| { + search.as_deref().is_none_or(|needle| { + id.to_ascii_lowercase().contains(needle) + || name.is_some_and(|n| n.to_ascii_lowercase().contains(needle)) + }) }; - match self.client.get::(&path).await { - Ok(resp) => Ok(ToolResult::success_with_markdown( - resp.clone(), - serde_json::to_string_pretty(&resp).unwrap_or_else(|_| resp.to_string()), - )), - Err(e) => Ok(ToolResult::error(format!( - "Failed to list media models: {e}" - ))), + let mut out = serde_json::Map::new(); + if kind != "video" { + match self.image.list_models().await { + Ok(models) => { + let list: Vec = models + .iter() + .filter(|m| keep(&m.id, m.name.as_deref())) + .map(|m| json!({ "id": m.id, "name": m.name })) + .collect(); + out.insert( + "image".into(), + json!({ "default": self.image.default_model(), "models": list }), + ); + } + Err(error) => { + return Ok(ToolResult::error(format!( + "Listing image models failed: {error}" + ))) + } + } } + if kind != "image" { + match self.video.list_models().await { + Ok(models) => { + let list: Vec = models + .iter() + .filter(|m| keep(&m.id, m.name.as_deref())) + .map(|m| json!({ "id": m.id, "name": m.name })) + .collect(); + out.insert( + "video".into(), + json!({ "default": self.video.default_model(), "models": list }), + ); + } + Err(error) => { + return Ok(ToolResult::error(format!( + "Listing video models failed: {error}" + ))) + } + } + } + Ok(ToolResult::json(Value::Object(out))) } } -// ── Builder ───────────────────────────────────────────────────────── - -/// Build the media-generation tool surface. Returns empty when no integration -/// client is configured (no backend URL / not signed in), mirroring the other -/// backend-proxied tool families. -pub fn build_media_tools(root_config: &Config, action_dir: &std::path::Path) -> Vec> { - let Some(client) = crate::integrations::build_client(root_config) else { - tracing::debug!("[media_generation] no integration client — media tools skipped"); - return Vec::new(); - }; - - let action_dir = action_dir.to_path_buf(); - let tools: Vec> = vec![ - Box::new(MediaGenerateImageTool::new( - Arc::clone(&client), - action_dir.clone(), - )), - Box::new(MediaGenerateVideoTool::new(Arc::clone(&client), action_dir)), - Box::new(MediaListModelsTool::new(Arc::clone(&client))), - ]; - tracing::debug!("[media_generation] registered {} media tools", tools.len()); - tools -} - #[cfg(test)] #[path = "tools_tests.rs"] mod tools_tests; diff --git a/crates/openhuman-core/src/media/generation/tools_tests.rs b/crates/openhuman-core/src/media/generation/tools_tests.rs index ccf5e66a3a..b0a5355c72 100644 --- a/crates/openhuman-core/src/media/generation/tools_tests.rs +++ b/crates/openhuman-core/src/media/generation/tools_tests.rs @@ -1,336 +1,157 @@ -use std::path::PathBuf; +use std::path::Path; use std::sync::Arc; use serde_json::json; - -use super::{MediaGenerateImageTool, MediaGenerateVideoTool, MediaListModelsTool}; -use crate::integrations::IntegrationClient; +use tinyagents_harness::tinyinference_image::MockImageGenerator; +use tinyagents_harness::tinyinference_video::{MockVideoGenerator, MockVideoScript}; use tinytools::{PermissionLevel, Tool, ToolCategory}; -fn dummy_client() -> Arc { - // No requests are made in these tests; the URL/token are placeholders. - Arc::new(IntegrationClient::new( - "http://127.0.0.1:0".to_string(), - "test-token".to_string(), - )) +use super::{ + media_tools_from, reference_policy, IMAGE_TOOL_NAME, LIST_MODELS_TOOL_NAME, VIDEO_TOOL_NAME, +}; +use crate::media::generation::MediaGenerators; + +fn tools(action_dir: &Path) -> Vec> { + media_tools_from( + MediaGenerators { + image: Arc::new(MockImageGenerator::new()), + video: Arc::new(MockVideoGenerator::new(MockVideoScript::delivers())), + }, + action_dir, + &action_dir.join("workspace"), + tinyagents_harness::tinyinference_video::WaitPolicy::new( + std::time::Duration::from_millis(1), + std::time::Duration::from_secs(5), + ), + ) +} + +fn by_name<'a>(tools: &'a [Box], name: &str) -> &'a dyn Tool { + tools + .iter() + .find(|tool| tool.name() == name) + .map(AsRef::as_ref) + .unwrap_or_else(|| panic!("missing tool {name}")) } +/// The `media` tool pack and the image/video agents' allowlists pin these +/// names; renaming a tool silently drops it from every agent. #[test] -fn image_tool_schema_and_metadata() { - let tool = MediaGenerateImageTool::new(dummy_client(), PathBuf::from("/tmp")); - assert_eq!(tool.name(), "media_generate_image"); - assert_eq!(tool.permission_level(), PermissionLevel::Execute); - assert_eq!(tool.category(), ToolCategory::Workflow); +fn tool_names_match_the_media_pack() { + let dir = tempfile::tempdir().unwrap(); + let tools = tools(dir.path()); + let names: Vec<&str> = tools.iter().map(|tool| tool.name()).collect(); + assert_eq!( + names, + vec![IMAGE_TOOL_NAME, VIDEO_TOOL_NAME, LIST_MODELS_TOOL_NAME] + ); + assert_eq!( + names, + vec![ + "media_generate_image", + "media_generate_video", + "media_list_models" + ] + ); +} - let schema = tool.parameters_schema(); - assert_eq!(schema["required"], json!(["prompt"])); - let props = schema["properties"].as_object().unwrap(); - for key in ["prompt", "model", "size", "n", "input_images", "seed"] { - assert!(props.contains_key(key), "missing image property {key}"); +#[test] +fn generation_tools_keep_their_host_metadata() { + let dir = tempfile::tempdir().unwrap(); + let tools = tools(dir.path()); + for name in [IMAGE_TOOL_NAME, VIDEO_TOOL_NAME] { + let tool = by_name(&tools, name); + assert_eq!(tool.permission_level(), PermissionLevel::Execute, "{name}"); + assert_eq!(tool.category(), ToolCategory::Workflow, "{name}"); + assert!(tool.external_effect(), "{name}"); + assert!(tool.policy().side_effects.payment, "{name} is billed"); } + let list = by_name(&tools, LIST_MODELS_TOOL_NAME); + assert_eq!(list.permission_level(), PermissionLevel::ReadOnly); } #[test] -fn video_tool_schema_and_metadata() { - let tool = MediaGenerateVideoTool::new(dummy_client(), PathBuf::from("/tmp")); - assert_eq!(tool.name(), "media_generate_video"); - assert_eq!(tool.permission_level(), PermissionLevel::Execute); - assert_eq!(tool.category(), ToolCategory::Workflow); - - let schema = tool.parameters_schema(); - assert_eq!(schema["required"], json!(["prompt"])); - let props = schema["properties"].as_object().unwrap(); +fn schemas_expose_the_reference_standards() { + let dir = tempfile::tempdir().unwrap(); + let tools = tools(dir.path()); + let image = by_name(&tools, IMAGE_TOOL_NAME).parameters_schema(); + assert_eq!(image["required"], json!(["prompt"])); for key in [ "prompt", "model", - "input_image", - "duration_seconds", + "n", "aspect_ratio", - "negative_prompt", + "resolution", + "size", "seed", + "references", ] { - assert!(props.contains_key(key), "missing video property {key}"); + assert!( + image["properties"].get(key).is_some(), + "image schema missing {key}" + ); + } + let video = by_name(&tools, VIDEO_TOOL_NAME).parameters_schema(); + for key in [ + "prompt", + "duration", + "resolution", + "aspect_ratio", + "first_frame", + "last_frame", + "references", + "resume_job_id", + ] { + assert!( + video["properties"].get(key).is_some(), + "video schema missing {key}" + ); } -} - -#[test] -fn list_models_tool_metadata() { - let tool = MediaListModelsTool::new(dummy_client()); - assert_eq!(tool.name(), "media_list_models"); - assert_eq!(tool.category(), ToolCategory::Workflow); - assert!(tool.parameters_schema()["properties"] - .as_object() - .unwrap() - .contains_key("include_upstream")); -} - -#[tokio::test] -async fn image_tool_rejects_empty_prompt_without_network() { - let tool = MediaGenerateImageTool::new(dummy_client(), PathBuf::from("/tmp")); - let result = tool.execute(json!({ "prompt": " " })).await.unwrap(); - assert!(result.is_error); -} - -#[tokio::test] -async fn video_tool_rejects_missing_prompt_without_network() { - let tool = MediaGenerateVideoTool::new(dummy_client(), PathBuf::from("/tmp")); - let result = tool.execute(json!({ "model": "x" })).await.unwrap(); - assert!(result.is_error); -} - -// ── End-to-end flow against a mock backend (wiremock) ─────────────── - -use wiremock::matchers::{method, path, path_regex}; -use wiremock::{Mock, MockServer, ResponseTemplate}; - -fn client_for(server: &MockServer) -> std::sync::Arc { - std::sync::Arc::new(IntegrationClient::new(server.uri(), "tok".to_string())) -} - -/// Mount a media download endpoint that returns `bytes` for the given path. -async fn mount_media(server: &MockServer, p: &str, content_type: &str, bytes: &[u8]) { - Mock::given(method("GET")) - .and(path(p.to_string())) - .respond_with(ResponseTemplate::new(200).set_body_raw(bytes.to_vec(), content_type)) - .mount(server) - .await; } #[tokio::test] -async fn image_tool_submits_downloads_and_persists_local_artifact() { - let server = MockServer::start().await; - let media_url = format!("{}/media/out.png", server.uri()); - Mock::given(method("POST")) - .and(path("/agent-integrations/media-generation/images")) - .respond_with(ResponseTemplate::new(200).set_body_json( - serde_json::json!({ "success": true, "data": { - "requestId": "req-1", - "status": "success", - "model": "seedream-4-0-250828", - "media": [{ "type": "image", "url": media_url }], - "costUsd": 0.039 - } }), - )) - .mount(&server) - .await; - mount_media(&server, "/media/out.png", "image/png", b"PNGBYTES").await; - - let tmp = tempfile::tempdir().unwrap(); - let tool = MediaGenerateImageTool::new(client_for(&server), tmp.path().to_path_buf()); - let res = tool - .execute(json!({ "prompt": "a fox", "size": "1024x1024" })) +async fn image_tool_saves_under_generated_media_in_the_action_dir() { + let dir = tempfile::tempdir().unwrap(); + let tools = tools(dir.path()); + let result = by_name(&tools, IMAGE_TOOL_NAME) + .execute(json!({ "prompt": "an anime comic about a delivery certificate" })) .await .unwrap(); - - assert!(!res.is_error, "expected success, got {res:?}"); - let dir = tmp.path().join("generated-media"); - let files: Vec<_> = std::fs::read_dir(&dir) + assert!(!result.is_error, "{result:?}"); + let saved = std::fs::read_dir(dir.path().join("generated-media")) .unwrap() - .filter_map(Result::ok) - .collect(); - assert_eq!(files.len(), 1, "exactly one artifact should be persisted"); - assert_eq!(std::fs::read(files[0].path()).unwrap(), b"PNGBYTES"); + .count(); + assert_eq!(saved, 1); } #[tokio::test] -async fn video_tool_persists_clip_with_image_to_video_payload() { - let server = MockServer::start().await; - let media_url = format!("{}/media/clip.mp4", server.uri()); - Mock::given(method("POST")) - .and(path("/agent-integrations/media-generation/videos")) - .respond_with(ResponseTemplate::new(200).set_body_json( - serde_json::json!({ "success": true, "data": { - "requestId": "vid-1", - "status": "success", - "model": "seedance-1-0-pro-fast-251015", - "media": [{ "type": "video", "url": media_url, "thumbnailUrl": "https://x/t.png" }], - "costUsd": 0.13 - } }), - )) - .mount(&server) - .await; - mount_media(&server, "/media/clip.mp4", "video/mp4", b"MP4BYTES").await; - - let tmp = tempfile::tempdir().unwrap(); - let tool = MediaGenerateVideoTool::new(client_for(&server), tmp.path().to_path_buf()); - let res = tool - .execute( - json!({ "prompt": "a wave", "input_image": "https://in/f.png", "duration_seconds": 6 }), - ) +async fn list_models_reports_both_catalogs_and_defaults() { + let dir = tempfile::tempdir().unwrap(); + let tools = tools(dir.path()); + let result = by_name(&tools, LIST_MODELS_TOOL_NAME) + .execute(json!({})) .await .unwrap(); - - assert!(!res.is_error, "expected success, got {res:?}"); - let dir = tmp.path().join("generated-media"); - let files: Vec<_> = std::fs::read_dir(&dir) - .unwrap() - .filter_map(Result::ok) - .collect(); - assert_eq!(files.len(), 1); - assert!(files[0].path().extension().is_some_and(|e| e == "mp4")); -} - -#[tokio::test] -async fn image_tool_polls_until_terminal_then_persists() { - let server = MockServer::start().await; - let media_url = format!("{}/media/p.png", server.uri()); - // Submit returns a non-terminal status; the tool must poll the status endpoint. - Mock::given(method("POST")) - .and(path("/agent-integrations/media-generation/images")) - .respond_with(ResponseTemplate::new(200).set_body_json( - serde_json::json!({ "success": true, "data": { - "requestId": "req-2", "status": "queued", "model": "seedream-4-0-250828", "media": [] - } }), - )) - .mount(&server) - .await; - Mock::given(method("GET")) - .and(path_regex( - r"^/agent-integrations/media-generation/requests/.+", - )) - .respond_with(ResponseTemplate::new(200).set_body_json( - serde_json::json!({ "success": true, "data": { - "requestId": "req-2", - "status": "success", - "model": "seedream-4-0-250828", - "media": [{ "type": "image", "url": media_url }], - "costUsd": 0.039 - } }), - )) - .mount(&server) - .await; - mount_media(&server, "/media/p.png", "image/png", b"POLLED").await; - - let tmp = tempfile::tempdir().unwrap(); - let tool = MediaGenerateImageTool::new(client_for(&server), tmp.path().to_path_buf()); - let res = tool.execute(json!({ "prompt": "a fox" })).await.unwrap(); - assert!(!res.is_error, "expected success after poll, got {res:?}"); - assert_eq!( - std::fs::read_dir(tmp.path().join("generated-media")) - .unwrap() - .count(), - 1 - ); -} - -#[tokio::test] -async fn image_tool_reports_failed_terminal_status() { - let server = MockServer::start().await; - Mock::given(method("POST")) - .and(path("/agent-integrations/media-generation/images")) - .respond_with(ResponseTemplate::new(200).set_body_json( - serde_json::json!({ "success": true, "data": { - "requestId": "req-3", "status": "failed", "model": "seedream-4-0-250828", "media": [] - } }), - )) - .mount(&server) - .await; - let tmp = tempfile::tempdir().unwrap(); - let tool = MediaGenerateImageTool::new(client_for(&server), tmp.path().to_path_buf()); - let res = tool.execute(json!({ "prompt": "a fox" })).await.unwrap(); - assert!(res.is_error); -} - -#[tokio::test] -async fn list_models_tool_returns_backend_catalog() { - let server = MockServer::start().await; - Mock::given(method("GET")) - .and(path("/agent-integrations/media-generation/models")) - .respond_with(ResponseTemplate::new(200).set_body_json( - serde_json::json!({ "success": true, "data": { - "curated": [{ "id": "seedream-4-0-250828", "modality": "image" }] - } }), - )) - .mount(&server) - .await; - let tool = MediaListModelsTool::new(client_for(&server)); - let res = tool.execute(json!({})).await.unwrap(); - assert!(!res.is_error, "expected success, got {res:?}"); -} - -#[tokio::test] -async fn deadline_without_terminal_status_errors_without_persisting() { - let server = MockServer::start().await; - // Submit is accepted but the request never reaches a terminal state. With a - // zero-second wait budget the poll deadline is hit immediately, so nothing is - // ever downloaded — the tool must surface an error, not a false success. - Mock::given(method("POST")) - .and(path("/agent-integrations/media-generation/images")) - .respond_with(ResponseTemplate::new(200).set_body_json( - serde_json::json!({ "success": true, "data": { - "requestId": "req-timeout", - "status": "queued", - "model": "seedream-4-0-250828", - "media": [] - } }), - )) - .mount(&server) - .await; - - let tmp = tempfile::tempdir().unwrap(); - let client = client_for(&server); - let res = super::generate_and_persist( - &client, - tmp.path(), - super::IMAGES_PATH, - json!({ "prompt": "a fox", "wait": false }), - 0, - ) - .await; - + let text = serde_json::to_string(&result).unwrap(); assert!( - res.is_error, - "deadline with no terminal status must error, got {res:?}" - ); - let dir = tmp.path().join("generated-media"); - assert!( - !dir.exists() || std::fs::read_dir(&dir).unwrap().count() == 0, - "no artifact should be persisted on a timeout" + text.contains("mock/image") && text.contains("mock/video"), + "{text}" ); } -#[tokio::test] -async fn deadline_after_poll_errors_still_errors() { - let server = MockServer::start().await; - // Submit is accepted but stays non-terminal, and every status poll fails. - // Transient poll errors must not abort the paid generation, but once the wait - // budget elapses the tool must surface an error (never a false success), - // remembering the last poll failure. The 1s budget caps the first sleep to the - // remaining time (not the full 4s interval), so exactly one failing poll runs - // before the deadline fires. - Mock::given(method("POST")) - .and(path("/agent-integrations/media-generation/images")) - .respond_with(ResponseTemplate::new(200).set_body_json( - serde_json::json!({ "success": true, "data": { - "requestId": "req-pollerr", - "status": "queued", - "model": "seedream-4-0-250828", - "media": [] - } }), - )) - .mount(&server) - .await; - Mock::given(method("GET")) - .and(path_regex( - r"^/agent-integrations/media-generation/requests/.+", - )) - .respond_with(ResponseTemplate::new(500)) - .mount(&server) - .await; - - let tmp = tempfile::tempdir().unwrap(); - let client = client_for(&server); - let res = super::generate_and_persist( - &client, - tmp.path(), - super::IMAGES_PATH, - json!({ "prompt": "a fox", "wait": false }), - 1, - ) - .await; - +#[test] +fn reference_policy_admits_workspace_files_only() { + let action = Path::new("/home/user/OpenHuman/projects"); + let workspace = Path::new("/home/user/.openhuman/users/u/workspace"); + let policy = reference_policy(action, workspace); + + assert!(policy(&action.join("art/ref.png")).is_ok()); + assert!(policy(&workspace.join("attachments/photo.jpg")).is_ok()); + assert!(policy(&action.join("../secret.png")).is_err()); + assert!(policy(Path::new("/etc/passwd")).is_err()); + assert!(policy(Path::new("/home/user/Documents/private.png")).is_err()); assert!( - res.is_error, - "deadline after failing polls must error, got {res:?}" + policy(&action.join(".ssh/id_rsa")).is_err(), + "credential stores stay forbidden" ); } diff --git a/crates/openhuman-core/src/media/generation/types.rs b/crates/openhuman-core/src/media/generation/types.rs deleted file mode 100644 index be15591fbf..0000000000 --- a/crates/openhuman-core/src/media/generation/types.rs +++ /dev/null @@ -1,47 +0,0 @@ -//! Shared types for the `media_generation` agent tools. -//! -//! These mirror the backend's standardized `media_generation` contract -//! (`/agent-integrations/media-generation/*`) — see -//! `backend/docs/media-generation.md`. The backend normalizes GMI's per-model -//! payload/outcome shapes; the core only depends on this stable envelope. - -use serde::Deserialize; - -/// A single generated artifact as returned by the backend. The `url` is an -/// expiring signed URL — the core downloads + persists it locally. -#[derive(Debug, Clone, Deserialize)] -pub struct MediaItem { - #[serde(rename = "type")] - pub kind: String, - pub url: String, - #[serde(rename = "thumbnailUrl", default)] - pub thumbnail_url: Option, -} - -/// Standardized media-generation response envelope. -#[derive(Debug, Clone, Deserialize)] -pub struct MediaResponse { - #[serde(rename = "requestId")] - pub request_id: String, - pub status: String, - #[serde(default)] - pub model: String, - #[serde(default)] - pub media: Vec, - #[serde(rename = "costUsd", default)] - pub cost_usd: f64, -} - -impl MediaResponse { - pub fn is_success(&self) -> bool { - self.status.eq_ignore_ascii_case("success") - } - - pub fn is_failed(&self) -> bool { - self.status.eq_ignore_ascii_case("failed") - } - - pub fn is_terminal(&self) -> bool { - self.is_success() || self.is_failed() - } -} diff --git a/crates/openhuman-core/src/tools/toolpacks/tools.rs b/crates/openhuman-core/src/tools/toolpacks/tools.rs index e5ed4c75ff..fece5d6d80 100644 --- a/crates/openhuman-core/src/tools/toolpacks/tools.rs +++ b/crates/openhuman-core/src/tools/toolpacks/tools.rs @@ -112,6 +112,25 @@ impl PackRegistryHandle { self.find(tool) } + /// Resolves `tool` in `skill`'s pack, returning the exact registry `Arc` + /// it lives in — not a clone of the tool itself — so a caller can re-wrap + /// it in the same `CanonicalSharedToolAdapter` seam the harness uses at + /// registration for typed-dispatch selection. + /// + /// `pub(crate)`, not private: `use_skill`'s typed dispatch + /// (`agent::tinyagents::use_skill_dispatch::UseSkillDispatch`) needs the + /// same resolution [`UseSkillTool::execute_with_context`] performs, so a + /// packed archetype delegation reached through `use_skill` can be + /// re-dispatched through the live-parent typed-dispatch seam instead of + /// falling back to plain `Tool::execute_with_context` (regression R3). + pub(crate) fn resolve_registry_for( + &self, + skill: &str, + tool: &str, + ) -> Option>>> { + self.resolve(skill, tool).map(|(tools, _idx)| tools) + } + /// Locate `tool` in whichever registry holds it. fn find(&self, tool: &str) -> Option<(ToolVec, usize)> { for tools in self.registries() { diff --git a/docs/TEST-COVERAGE-MATRIX.md b/docs/TEST-COVERAGE-MATRIX.md index 32432e5e22..d80ffe855d 100644 --- a/docs/TEST-COVERAGE-MATRIX.md +++ b/docs/TEST-COVERAGE-MATRIX.md @@ -203,6 +203,7 @@ Canonical mapping of every product feature to its test source(s). Drives gap-fil | 4.3.3 | Tool Failure Handling | WD | `skill-execution-flow.spec.ts` | ✅ | | | 4.3.4 | Subagent Mascot Visualization | VU | `app/src/features/human/SubMascotLayer.test.tsx`, `app/src/features/human/chatMascot/ChatMascotOverlay.test.tsx` | ✅ | Renders spawned/completed/failed subagent timeline rows as colored companion mascots with activity bubbles | | 4.3.5 | Image Tool Contracts | RU | `crates/openhuman-core/src/media/image/` | ✅ | High-level `image_generation` / `view_image` schema, gating, serialization, prompt guidance, and contract e2e coverage for #2984 | +| 4.3.6 | Media generation (OpenRouter via backend proxy) | RU+E2E | `crates/openhuman-core/src/media/generation/tools_tests.rs`, `tests/media_generation_e2e.rs::media_tools_deliver_images_and_videos_through_the_backend_proxy`, `scripts/mock-api/routes/__tests__/media.test.mjs` | ✅ | `media_generate_image` / `media_generate_video` run TinyInference generators through `/agent-integrations/openrouter` (envelope, credential, `x-sdk-name`); a video job reporting `completed` with no outputs is polled through, not failed; one billed submit per call; local references confined to action/workspace dirs. | | 4.3.7 | Mascot Avatar Animation | VU | `app/src/features/human/Mascot/RiveMascot.test.tsx`, `app/src/features/human/Mascot/riveMaps.test.ts` | ✅ | Rive `MascotSM` state machine: face→pose mapping, Oculus→`visme_codes` viseme normalization, and idle random pose rotation for the `tiny_mascot.riv` upgrade | ### 4.4 Agent Harness Behaviors @@ -287,6 +288,7 @@ End-to-end coverage of the agent harness via the web-chat RPC surface against an | 6.3.16 | Sub-agent spawn/delegate tool refusal (#4452 invariant) | RU | `crates/openhuman-core/src/agent/subagent_host/ops/graph_tests.rs::{a_sub_agent_cannot_reach_a_spawn_tool_and_the_healthy_run_is_quiet,an_allowlist_that_readmits_a_spawn_tool_is_refused_loudly}`, `crates/openhuman-core/src/agent/subagent_host/tool_prep_tests.rs::{dynamic_tools_keep_ordinary_actions_and_lose_spawn_tools,dynamic_tools_lose_unprefixed_delegate_name_overrides,a_dynamic_tool_list_without_spawn_tools_is_untouched}` | 🟡 | Migration coverage: the host still fail-closes `spawn_subagent`/`delegate_*`/`agent_prepare_context`/`spawn_worker_thread` while Phase 6 moves lifecycle ownership to TinyAgents. Completion requires the direct-driver and durable-resume checks in the extraction plan. | | 6.3.17 | Host-authored turns run on the thread's cached session (no competing root transcript) | RU | `crates/openhuman-core/src/web_chat/session_checkout_tests.rs::{checkout_cold_boots_from_the_thread_transcript_and_checkin_keeps_it_warm,checkin_if_vacant_yields_to_a_turn_that_re_cached_meanwhile,a_system_turn_adopts_the_cached_agent_and_its_fingerprint,a_fork_never_takes_or_returns_the_cached_agent}` | ✅ | Background delivery and goal continuation go through `web_chat::run_system_turn_on_thread` → `checkout_session_agent`, so they see the conversation and append to the thread's transcript. A throwaway host bound to the thread used to write a competing root transcript that the next cold-boot resume preferred (newest `created`), dropping every earlier turn after a restart. `checkin_session_agent_if_vacant` never clobbers a user turn that re-cached meanwhile; forks stay isolated. | | 6.3.18 | Mid-conversation availability notes are status, not instructions | RU | `crates/openhuman-core/src/agent/session_host/runtime_adapter_tests.rs::availability_notes_are_status_not_instructions` | ✅ | `[integration update]` / `[MCP update]` / `[skills update]` prepended to the next user message no longer say "act on them immediately" (which sent the orchestrator to the integrations agent mid-conversation); they defer to the user's message and only forbid the "reconnect/restart" reply. | +| 6.3.19 | `use_skill` dispatches packed delegates with the live parent | RU | `crates/openhuman-core/src/agent/tinyagents/use_skill_dispatch_tests.rs::use_skill_dispatch_reaches_live_parent_for_packed_archetype_delegate` | ✅ | `use_skill {skill, tool:"create_image"}` routes through the typed delegation dispatch with the real parent `RunContext` instead of failing with "delegation requires a live harness run context"; disclosure and not-found paths unchanged. | ### 6.4 Managed Cloud File Storage diff --git a/scripts/mock-api/routes/__tests__/media.test.mjs b/scripts/mock-api/routes/__tests__/media.test.mjs new file mode 100644 index 0000000000..387152e364 --- /dev/null +++ b/scripts/mock-api/routes/__tests__/media.test.mjs @@ -0,0 +1,92 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { handleMedia, resetMediaMock } from "../media.mjs"; + +function createRes() { + return { + statusCode: 0, + headers: {}, + body: "", + writeHead(status, headers = {}) { + this.statusCode = status; + this.headers = headers; + }, + setHeader(name, value) { + this.headers[name] = value; + }, + end(chunk = "") { + this.body += Buffer.isBuffer(chunk) + ? chunk.toString("latin1") + : String(chunk); + }, + }; +} + +function call(method, url, parsedBody) { + const res = createRes(); + const handled = handleMedia({ method, url, parsedBody, res }); + return { + handled, + res, + body: + res.headers["Content-Type"] === "video/mp4" + ? null + : JSON.parse(res.body || "null"), + }; +} + +test.beforeEach(() => resetMediaMock()); + +test("images return an enveloped OpenRouter body with base64 images", () => { + const { handled, res, body } = call( + "POST", + "/agent-integrations/openrouter/images", + { prompt: "x", n: 2 }, + ); + assert.equal(handled, true); + assert.equal(res.statusCode, 200); + assert.equal(body.success, true); + assert.equal(body.data.data.length, 2); + assert.equal(body.data.data[0].media_type, "image/png"); + assert.ok(body.data.data[0].b64_json.length > 0); +}); + +test("images reject a missing prompt", () => { + const { res } = call("POST", "/agent-integrations/openrouter/images", {}); + assert.equal(res.statusCode, 400); +}); + +test("video jobs report completed-without-outputs before delivering", () => { + const submit = call("POST", "/agent-integrations/openrouter/videos", { + prompt: "x", + }); + const id = submit.body.data.id; + const poll = () => + call("GET", `/agent-integrations/openrouter/videos/${id}`).body.data; + + assert.equal(poll().status, "in_progress"); + const early = poll(); + assert.equal(early.status, "completed"); + assert.deepEqual( + early.unsigned_urls, + [], + "completed before the output exists", + ); + const done = poll(); + assert.equal(done.status, "completed"); + assert.equal(done.unsigned_urls.length, 1); + + const content = call( + "GET", + `/agent-integrations/openrouter/videos/${id}/content?index=0`, + ); + assert.equal(content.res.headers["Content-Type"], "video/mp4"); +}); + +test("unrelated routes fall through", () => { + assert.equal( + call("GET", "/agent-integrations/composio/tools").handled, + false, + ); +}); diff --git a/scripts/mock-api/routes/media.mjs b/scripts/mock-api/routes/media.mjs new file mode 100644 index 0000000000..72a1e56028 --- /dev/null +++ b/scripts/mock-api/routes/media.mjs @@ -0,0 +1,141 @@ +import { json } from "../http.mjs"; + +// OpenRouter media proxy (`/agent-integrations/openrouter/{images,videos}`), +// in the backend's `{success, data}` envelope around OpenRouter's own bodies. +// +// Video jobs deliberately report `completed` with NO `unsigned_urls` on their +// second poll before the output appears on the third — the shape that used to +// make the core give up on a billed, about-to-deliver generation. Clients must +// poll through it. + +const PREFIX = "/agent-integrations/openrouter"; + +// A 1×1 transparent PNG. +const PNG_BASE64 = + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGMAAQAABQABDQottAAAAABJRU5ErkJggg=="; +// A minimal MP4 `ftyp` box — enough for type sniffing. +const MP4_BYTES = Buffer.from("AAAAGGZ0eXBtcDQyAAAAAG1wNDJpc29t", "base64"); + +/** Poll counts per job id, so each job walks the scripted lifecycle once. */ +const pollsByJob = new Map(); +let nextJob = 1; + +/** Resets job state between tests. */ +export function resetMediaMock() { + pollsByJob.clear(); + nextJob = 1; +} + +function jobStatus(jobId) { + const polls = (pollsByJob.get(jobId) ?? 0) + 1; + pollsByJob.set(jobId, polls); + if (polls === 1) return { status: "in_progress", unsigned_urls: [] }; + if (polls === 2) return { status: "completed", unsigned_urls: [] }; + return { + status: "completed", + unsigned_urls: [`https://cdn.mock/${jobId}/0.mp4`], + usage: { cost: 0.12 }, + }; +} + +export function handleMedia(ctx) { + const { method, url, parsedBody, res } = ctx; + if (!url.startsWith(PREFIX)) return false; + const path = url.slice(PREFIX.length).split("?")[0]; + + if (method === "GET" && path === "/images/models") { + json(res, 200, { + success: true, + data: { + object: "list", + data: [ + { + id: "bytedance-seed/seedream-5-0-lite", + display_name: "Seedream 5.0 Lite", + supported_parameters: { + aspect_ratio: { + type: "enum", + values: ["1:1", "16:9", "9:16", "4:3", "3:4", "auto"], + }, + n: { type: "range", min: 1, max: 4 }, + input_references: { type: "range", min: 0, max: 14 }, + seed: { type: "boolean" }, + }, + }, + ], + }, + }); + return true; + } + + if (method === "POST" && path === "/images") { + if (typeof parsedBody?.prompt !== "string" || !parsedBody.prompt.trim()) { + json(res, 400, { success: false, error: "prompt is required" }); + return true; + } + const n = Math.max(1, Math.min(4, Number(parsedBody.n ?? 1))); + json(res, 200, { + success: true, + data: { + created: Math.floor(Date.now() / 1000), + data: Array.from({ length: n }, () => ({ + b64_json: PNG_BASE64, + media_type: "image/png", + })), + usage: { cost: 0.035 * n }, + }, + }); + return true; + } + + if (method === "GET" && path === "/videos/models") { + json(res, 200, { + success: true, + data: { + object: "list", + data: [ + { + id: "bytedance/seedance-2.0-mini", + display_name: "Seedance 2.0 Mini", + supported_resolutions: ["480p", "720p"], + supported_durations: [4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], + supported_frame_images: ["first_frame", "last_frame"], + generate_audio: true, + seed: true, + }, + ], + }, + }); + return true; + } + + if (method === "POST" && path === "/videos") { + const id = `gen-vid-1790000000-mock${String(nextJob++).padStart(16, "0")}`; + json(res, 200, { + success: true, + data: { id, polling_url: `/api/v1/videos/${id}`, status: "pending" }, + }); + return true; + } + + const content = path.match(/^\/videos\/([^/]+)\/content$/); + if (method === "GET" && content) { + res.writeHead(200, { + "Content-Type": "video/mp4", + "Content-Length": MP4_BYTES.length, + }); + res.end(MP4_BYTES); + return true; + } + + const job = path.match(/^\/videos\/([^/]+)$/); + if (method === "GET" && job) { + json(res, 200, { + success: true, + data: { id: job[1], ...jobStatus(job[1]) }, + }); + return true; + } + + return false; +} diff --git a/scripts/mock-api/server.mjs b/scripts/mock-api/server.mjs index b3c15ee485..ea13a41f35 100644 --- a/scripts/mock-api/server.mjs +++ b/scripts/mock-api/server.mjs @@ -16,6 +16,7 @@ import { handleCron } from "./routes/cron.mjs"; import { handleIntegrations } from "./routes/integrations.mjs"; import { handleInvites } from "./routes/invites.mjs"; import { handleLlmCompletions, handleModelListing } from "./routes/llm.mjs"; +import { handleMedia } from "./routes/media.mjs"; import { handleOAuth } from "./routes/oauth.mjs"; import { handlePayments } from "./routes/payments.mjs"; import { handleTelegram } from "./routes/telegram.mjs"; @@ -53,6 +54,8 @@ const ROUTE_HANDLERS = [ // the default "Hello from e2e mock agent" reply. handleLlmCompletions, handleModelListing, + // OpenRouter media proxy; before the generic integrations handler. + handleMedia, handleIntegrations, handleWebhooks, handleCron, diff --git a/tests/media_generation_e2e.rs b/tests/media_generation_e2e.rs new file mode 100644 index 0000000000..6d8fddf64e --- /dev/null +++ b/tests/media_generation_e2e.rs @@ -0,0 +1,263 @@ +//! End-to-end regression for media generation (the "anime cartoon" incident). +//! +//! Boots the real TinyHumans backend transport in-process, builds the +//! production `media_generate_image` / `media_generate_video` tools through +//! `build_media_tools`' own code path (`managed_generators` + +//! `media_tools_from`), and drives them against a scripted fake of the +//! backend's `/agent-integrations/openrouter` proxy: +//! +//! - responses arrive in the backend's `{success, data}` envelope; +//! - the video job reports `completed` with **no** `unsigned_urls` before the +//! output exists — the exact shape that previously ended in "reported +//! success but returned no media" while the generation was billed; +//! - the clip is downloaded through the authenticated content proxy. +//! +//! It asserts the files land in `generated-media/`, that every request carried +//! the backend credential and the product-identity header, and that the job +//! was polled through the empty `completed` state rather than failing on it. + +#![cfg(feature = "media")] + +#[path = "support/tinyhumans_boot.rs"] +mod tinyhumans_boot; + +use std::path::Path; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use axum::extract::{Path as AxumPath, State}; +use axum::http::HeaderMap; +use axum::response::IntoResponse; +use axum::routing::{get, post}; +use axum::{Json, Router}; +use base64::Engine as _; +use serde_json::{json, Value}; +use tinyagents_harness::tinyinference_video::WaitPolicy; +use tinytools::Tool; + +use openhuman_core::config::Config; +use openhuman_core::media::generation::{managed_generators, media_tools_from}; + +/// A 1×1 PNG. +const PNG: &[u8] = &[ + 0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, 0x00, 0x00, 0x00, 0x0D, 0x49, 0x48, 0x44, 0x52, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x08, 0x06, 0x00, 0x00, 0x00, 0x1F, 0x15, 0xC4, + 0x89, 0x00, 0x00, 0x00, 0x0D, 0x49, 0x44, 0x41, 0x54, 0x78, 0x9C, 0x63, 0x00, 0x01, 0x00, 0x00, + 0x05, 0x00, 0x01, 0x0D, 0x0A, 0x2D, 0xB4, 0x00, 0x00, 0x00, 0x00, 0x49, 0x45, 0x4E, 0x44, 0xAE, + 0x42, 0x60, 0x82, +]; +const MP4: &[u8] = b"\x00\x00\x00\x18ftypmp42\x00\x00\x00\x00mp42isom"; +const API_KEY: &str = "th_live_media_e2e_0123456789abcdef"; + +#[derive(Clone, Default)] +struct Backend { + /// `(method path, authorization, x-sdk-name present)` per request. + requests: Arc>>, + image_bodies: Arc>>, + video_bodies: Arc>>, + polls: Arc, +} + +impl Backend { + fn record(&self, route: &str, headers: &HeaderMap) { + let auth = headers + .get("authorization") + .and_then(|v| v.to_str().ok()) + .unwrap_or_default() + .to_owned(); + let sdk_name = headers.contains_key("x-sdk-name"); + self.requests + .lock() + .unwrap() + .push((route.to_owned(), auth, sdk_name)); + } +} + +async fn start_backend() -> (String, Backend) { + let backend = Backend::default(); + let prefix = "/agent-integrations/openrouter"; + let router = Router::new() + .route( + &format!("{prefix}/images"), + post(|State(b): State, headers: HeaderMap, Json(body): Json| async move { + b.record("POST images", &headers); + b.image_bodies.lock().unwrap().push(body); + Json(json!({ "success": true, "data": { + "created": 1, + "data": [{ "b64_json": base64::engine::general_purpose::STANDARD.encode(PNG), "media_type": "image/png" }], + "usage": { "cost": 0.035 } + }})) + }), + ) + .route( + &format!("{prefix}/images/models"), + get(|State(b): State, headers: HeaderMap| async move { + b.record("GET images/models", &headers); + Json(json!({ "success": true, "data": { "object": "list", "data": [ + { "id": "bytedance-seed/seedream-5-0-lite", "display_name": "Seedream 5.0 Lite" } + ]}})) + }), + ) + .route( + &format!("{prefix}/videos"), + post(|State(b): State, headers: HeaderMap, Json(body): Json| async move { + b.record("POST videos", &headers); + b.video_bodies.lock().unwrap().push(body); + Json(json!({ "success": true, "data": { + "id": "gen-vid-1790000000-abcdefghijklmnopqrst", + "polling_url": "/api/v1/videos/gen-vid-1790000000-abcdefghijklmnopqrst", + "status": "pending" + }})) + }), + ) + .route( + &format!("{prefix}/videos/models"), + get(|State(b): State, headers: HeaderMap| async move { + b.record("GET videos/models", &headers); + Json(json!({ "success": true, "data": { "object": "list", "data": [] } })) + }), + ) + .route( + &format!("{prefix}/videos/{{job}}"), + get( + |State(b): State, headers: HeaderMap, AxumPath(job): AxumPath| async move { + b.record("GET videos/:job", &headers); + let n = b.polls.fetch_add(1, Ordering::SeqCst); + // in_progress → completed WITHOUT outputs (the incident shape) + // → completed with the output. + let (status, urls) = match n { + 0 => ("in_progress", vec![]), + 1 | 2 => ("completed", vec![]), + _ => ("completed", vec!["https://cdn.example/out.mp4"]), + }; + Json(json!({ "success": true, "data": { + "id": job, "status": status, "unsigned_urls": urls, "usage": { "cost": 0.38 } + }})) + }, + ), + ) + .route( + &format!("{prefix}/videos/{{job}}/content"), + get(|State(b): State, headers: HeaderMap| async move { + b.record("GET videos/:job/content", &headers); + ([("content-type", "video/mp4")], MP4).into_response() + }), + ) + .with_state(backend.clone()); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + tokio::spawn(async move { + axum::serve(listener, router).await.ok(); + }); + (format!("http://{address}"), backend) +} + +fn config(root: &Path, api_url: &str) -> Config { + let mut config = Config::default(); + config.config_path = root.join("config.toml"); + config.workspace_dir = root.join("workspace"); + config.api_url = Some(api_url.to_owned()); + config.secrets.encrypt = false; + std::fs::create_dir_all(&config.workspace_dir).unwrap(); + openhuman_core::security::credentials::api_key::store_api_key(&config, API_KEY) + .expect("store the TinyHumans API key"); + config +} + +fn tool<'a>(tools: &'a [Box], name: &str) -> &'a dyn Tool { + tools + .iter() + .find(|t| t.name() == name) + .map(AsRef::as_ref) + .unwrap_or_else(|| panic!("missing {name}")) +} + +#[tokio::test] +async fn media_tools_deliver_images_and_videos_through_the_backend_proxy() { + tinyhumans_boot::boot(); + let tmp = tempfile::tempdir().unwrap(); + let (api_url, backend) = start_backend().await; + let config = config(tmp.path(), &api_url); + let action_dir = tmp.path().join("projects"); + + let generators = managed_generators(&config).expect("backend transport is installed"); + let tools = media_tools_from( + generators, + &action_dir, + &config.workspace_dir, + WaitPolicy::new(Duration::from_millis(5), Duration::from_secs(20)), + ); + + // ── image ──────────────────────────────────────────────────────────── + let result = tool(&tools, "media_generate_image") + .execute(json!({ + "prompt": "a four-panel anime comic explaining a delivery certificate", + "aspect_ratio": "landscape", + "seed": 42 + })) + .await + .unwrap(); + assert!(!result.is_error, "image tool failed: {result:?}"); + + // ── video: must poll through `completed` with no outputs ───────────── + let result = tool(&tools, "media_generate_video") + .execute(json!({ + "prompt": "two engineers shake hands, anime style", + "duration": 4, + "resolution": "480", + "first_frame": "https://example.com/first.png" + })) + .await + .unwrap(); + assert!(!result.is_error, "video tool failed: {result:?}"); + assert!( + backend.polls.load(Ordering::SeqCst) >= 4, + "the job must be polled through the empty `completed` states" + ); + + // ── artifacts on disk ──────────────────────────────────────────────── + let mut saved: Vec<(String, Vec)> = std::fs::read_dir(action_dir.join("generated-media")) + .expect("generated-media directory") + .map(|entry| { + let path = entry.unwrap().path(); + ( + path.extension().unwrap().to_string_lossy().into_owned(), + std::fs::read(&path).unwrap(), + ) + }) + .collect(); + saved.sort(); + assert_eq!(saved.len(), 2, "one image and one video: {saved:?}"); + assert_eq!(saved[0].0, "mp4"); + assert_eq!(saved[0].1, MP4); + assert_eq!(saved[1].0, "png"); + assert_eq!(saved[1].1, PNG); + + // ── wire contract ──────────────────────────────────────────────────── + let image_body = backend.image_bodies.lock().unwrap()[0].clone(); + assert_eq!(image_body["model"], "bytedance-seed/seedream-5-0-lite"); + assert_eq!(image_body["aspect_ratio"], "16:9"); + assert_eq!(image_body["seed"], 42); + let video_body = backend.video_bodies.lock().unwrap()[0].clone(); + assert_eq!(video_body["model"], "bytedance/seedance-2.0-mini"); + assert_eq!(video_body["resolution"], "480p"); + assert_eq!(video_body["duration"], 4); + assert_eq!(video_body["frame_images"][0]["frame_type"], "first_frame"); + + // ── every request authenticated and attributed ─────────────────────── + let requests = backend.requests.lock().unwrap().clone(); + assert!(!requests.is_empty()); + for (route, auth, sdk_name) in &requests { + assert_eq!(auth, &format!("Bearer {API_KEY}"), "{route} credential"); + assert!(*sdk_name, "{route} is missing x-sdk-name"); + } + let submits = requests + .iter() + .filter(|(r, _, _)| r.starts_with("POST")) + .count(); + assert_eq!( + submits, 2, + "exactly one billed submit per tool call: {requests:?}" + ); +} diff --git a/vendor/tinyagents b/vendor/tinyagents index c7d3508a17..fcf7e884c4 160000 --- a/vendor/tinyagents +++ b/vendor/tinyagents @@ -1 +1 @@ -Subproject commit c7d3508a17678c0a1cf2d0ef4f4d3c3509718d7c +Subproject commit fcf7e884c407479721e03f08aa78a5166e44dd52 diff --git a/vendor/tinyhumans-sdk b/vendor/tinyhumans-sdk index 83ab7b1d32..f1e46de5b8 160000 --- a/vendor/tinyhumans-sdk +++ b/vendor/tinyhumans-sdk @@ -1 +1 @@ -Subproject commit 83ab7b1d32fdef85ec8d8427b92225a7575fb5cc +Subproject commit f1e46de5b83192b6db710028ac6212f4e026f79f