diff --git a/Cargo.lock b/Cargo.lock index a2edec8b83..d42fbeb16e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4104,8 +4104,8 @@ dependencies = [ "tinymemory-sources", "tinyruntime-bus", "tinyskills", - "tinytools 0.3.0", - "tinytools-agent 0.3.0", + "tinytools 0.4.1", + "tinytools-agent 0.4.1", "tinyvoice-bus", "tinywallet", "tinywallet-bus", @@ -4173,11 +4173,12 @@ dependencies = [ "tinyinference-core", "tinyinference-llm", "tinyinference-local", + "tinyjevclient", "tinymcp", "tinymcp-bus", "tinymemory-api", - "tinytools 0.3.0", - "tinytools-agent 0.3.0", + "tinytools 0.4.1", + "tinytools-agent 0.4.1", "tinytools-jev", "tokio", "tokio-stream", @@ -4239,7 +4240,8 @@ dependencies = [ "tempfile", "thiserror 2.0.20", "tinyhumans-sdk", - "tinytools 0.3.0", + "tinyjevclient", + "tinytools 0.4.1", "tinytools-jev", "tokio", "url", @@ -6439,7 +6441,7 @@ dependencies = [ "serde_json", "tinyagents-harness", "tinyinference-llm", - "tinytools 0.3.0", + "tinytools 0.4.1", "tokio", "tracing", ] @@ -6466,8 +6468,8 @@ dependencies = [ "tinyagents-definition", "tinyinference-embeddings", "tinyinference-llm", - "tinytools 0.3.0", - "tinytools-agent 0.3.0", + "tinytools 0.4.1", + "tinytools-agent 0.4.1", "tokio", "tracing", "uuid", @@ -6490,7 +6492,7 @@ dependencies = [ "tinyagents-runtime", "tinyagents-session", "tinyinference-llm", - "tinytools 0.3.0", + "tinytools 0.4.1", "tokio", "uuid", ] @@ -6506,7 +6508,7 @@ dependencies = [ "tinyagents-definition", "tinyagents-harness", "tinyinference-llm", - "tinytools 0.3.0", + "tinytools 0.4.1", ] [[package]] @@ -6519,7 +6521,7 @@ dependencies = [ "tinyagents-harness", "tinyagents-session", "tinyinference-llm", - "tinytools 0.3.0", + "tinytools 0.4.1", "tokio", ] @@ -6785,7 +6787,7 @@ dependencies = [ "sha2 0.11.0", "thiserror 2.0.20", "tinyinference-core", - "tinytools-agent 0.3.0 (git+https://github.com/tinyhumansai/tinytools?rev=a5d04a7f3af0abc1dd748f87f94871b36ae9fe71)", + "tinytools-agent 0.3.0", "tokio", "tracing", ] @@ -6850,7 +6852,7 @@ dependencies = [ [[package]] name = "tinyjevclient" version = "0.2.1" -source = "git+https://github.com/tinyhumansai/tinyjevclient?rev=e53d5f088ff03fa38c53bac219dab7697b5016c9#e53d5f088ff03fa38c53bac219dab7697b5016c9" +source = "git+https://github.com/tinyhumansai/tinyjevclient?rev=84b3983c7e1e14f7515658ceafabb6c4e7967c94#84b3983c7e1e14f7515658ceafabb6c4e7967c94" dependencies = [ "httpdate", "reqwest", @@ -6998,6 +7000,7 @@ dependencies = [ [[package]] name = "tinytools" version = "0.3.0" +source = "git+https://github.com/tinyhumansai/tinytools?rev=a5d04a7f3af0abc1dd748f87f94871b36ae9fe71#a5d04a7f3af0abc1dd748f87f94871b36ae9fe71" dependencies = [ "anyhow", "async-trait", @@ -7007,8 +7010,7 @@ dependencies = [ [[package]] name = "tinytools" -version = "0.3.0" -source = "git+https://github.com/tinyhumansai/tinytools?rev=a5d04a7f3af0abc1dd748f87f94871b36ae9fe71#a5d04a7f3af0abc1dd748f87f94871b36ae9fe71" +version = "0.4.1" dependencies = [ "anyhow", "async-trait", @@ -7019,6 +7021,7 @@ dependencies = [ [[package]] name = "tinytools-agent" version = "0.3.0" +source = "git+https://github.com/tinyhumansai/tinytools?rev=a5d04a7f3af0abc1dd748f87f94871b36ae9fe71#a5d04a7f3af0abc1dd748f87f94871b36ae9fe71" dependencies = [ "regex", "serde", @@ -7028,24 +7031,20 @@ dependencies = [ [[package]] name = "tinytools-agent" -version = "0.3.0" -source = "git+https://github.com/tinyhumansai/tinytools?rev=a5d04a7f3af0abc1dd748f87f94871b36ae9fe71#a5d04a7f3af0abc1dd748f87f94871b36ae9fe71" +version = "0.4.1" dependencies = [ "regex", "serde", "serde_json", - "tinytools 0.3.0 (git+https://github.com/tinyhumansai/tinytools?rev=a5d04a7f3af0abc1dd748f87f94871b36ae9fe71)", + "tinytools 0.4.1", ] [[package]] name = "tinytools-jev" -version = "0.3.0" +version = "0.4.1" dependencies = [ "async-trait", - "serde_json", - "tinyjevclient", - "tinytools 0.3.0", - "tokio", + "tinytools 0.4.1", ] [[package]] diff --git a/crates/openhuman-cli/Cargo.toml b/crates/openhuman-cli/Cargo.toml index 1fcc77a042..079680b8cc 100644 --- a/crates/openhuman-cli/Cargo.toml +++ b/crates/openhuman-cli/Cargo.toml @@ -345,6 +345,7 @@ tinytools-agent = { path = "../../vendor/tinyagents/vendor/tinytools/crates/tiny # The Jev ranker `tool-search-bench` measures; gated with the crate's `jev` # feature so the bin builds without a Jev client. tinytools-jev = { path = "../../vendor/tinyagents/vendor/tinytools/crates/tinytools-jev", optional = true } +tinyjevclient = { git = "https://github.com/tinyhumansai/tinyjevclient", rev = "84b3983c7e1e14f7515658ceafabb6c4e7967c94", optional = true } tokio = { version = "1", features = ["full"] } toml = "1.0" tracing = "0.1" @@ -393,7 +394,7 @@ wiremock = "0.6" default = ["openhuman-core/default", "openhuman-tinyhumans/default", "jev"] # The Jev-backed `tool_search` ranker; `openhuman-tinyhumans`'s own gate, plus # the client `tool-search-bench` measures it with. -jev = ["openhuman-tinyhumans/jev", "dep:tinytools-jev"] +jev = ["openhuman-tinyhumans/jev", "dep:tinytools-jev", "dep:tinyjevclient"] http-server = ["openhuman-core/http-server", "openhuman-tinyhumans/http-server"] inference = ["openhuman-core/inference", "openhuman-tinyhumans/inference"] documents = ["openhuman-core/documents", "openhuman-tinyhumans/documents"] diff --git a/crates/openhuman-cli/src/bin/README.md b/crates/openhuman-cli/src/bin/README.md index 10a9daaa01..a9fa708412 100644 --- a/crates/openhuman-cli/src/bin/README.md +++ b/crates/openhuman-cli/src/bin/README.md @@ -23,6 +23,7 @@ the arguments to `openhuman_core::run_core_from_args`. | `openhuman-fleet` | `fleet.rs` | `http-server`, `bin-tools` | Process-per-user supervisor + reverse proxy | | `rss-bench` | `rss_bench.rs` | `rss-bench` | Steady-state RSS benchmark for an embedded agent roster | | `tool-dialect-bench` | `tool_dialect_bench.rs` | none | Manual A/B of the text tool-call dialects against a local Ollama model | +| `tool-search-bench` | `tool_search_bench.rs` | none (`jev`, in `default`, for the Jev rankers) | `tool_search` ranker comparison (bm25 / overlap / embedding / jev) over the real orchestrator registry plus the recorded Composio catalogues, against `tests/fixtures/tool_search/intents.jsonl` | | `library-profile` | `library_profile/main.rs` (+ `harness.rs`, `mock.rs`, `scenarios/`) | `rss-bench` (add `rss-bench-dhat` for heap profiles) | Hermetic library-embedding profiling scenarios | `http-server` is in `default`; `bin-tools`, `rss-bench` and `rss-bench-dhat` diff --git a/crates/openhuman-cli/src/bin/tool_search_bench.rs b/crates/openhuman-cli/src/bin/tool_search_bench.rs index 7d5f2013eb..5c38894d86 100644 --- a/crates/openhuman-cli/src/bin/tool_search_bench.rs +++ b/crates/openhuman-cli/src/bin/tool_search_bench.rs @@ -112,6 +112,10 @@ struct RankerReport { usd: f64, /// `expected family -> top-1 family -> count`, labelled rows only. confusion: BTreeMap>, + /// `source -> (labelled, top1, top3, recall@k)` where source is `composio` + /// or `core`; the connector catalogue is the heavy one, so it is read on + /// its own. + by_source: BTreeMap, misses: Vec, } @@ -142,6 +146,8 @@ struct Args { top_k: usize, retrieval_k: usize, misses: bool, + family: bool, + embedding: bool, } fn parse_args() -> Args { @@ -153,6 +159,8 @@ fn parse_args() -> Args { top_k: 3, retrieval_k: 20, misses: false, + family: false, + embedding: false, }; let mut it = std::env::args().skip(1); while let Some(arg) = it.next() { @@ -166,10 +174,12 @@ fn parse_args() -> Args { args.retrieval_k = it.next().and_then(|v| v.parse().ok()).unwrap_or(20) } "--misses" => args.misses = true, + "--family" => args.family = true, + "--embedding" => args.embedding = true, "-h" | "--help" => { eprintln!( - "usage: tool-search-bench [--ranker all|bm25|overlap|jev] [--intents FILE] \ - [--dump-catalogue] [--json OUT] [--top-k N] [--retrieval-k N] [--misses]" + "usage: tool-search-bench [--ranker all|bm25|overlap|embedding|jev] [--intents FILE] \ + [--dump-catalogue] [--json OUT] [--top-k N] [--retrieval-k N] [--misses] [--family] [--embedding]" ); std::process::exit(0); } @@ -274,8 +284,14 @@ fn load_intents(path: &PathBuf) -> Result> { use openhuman_core::agent::tinyagents::discovery::OverlapRanker; #[cfg(feature = "jev")] -fn jev_ranker(retrieval_k: usize) -> Option<(Arc, Arc)> { - use tinytools_jev::{ClientConfig, JevRanker, JevRankerConfig}; +fn jev_ranker( + retrieval_k: usize, + family: bool, + embedding: bool, +) -> Option<(Arc, Arc)> { + use openhuman_tinyhumans::jev::TinyJevEvaluator; + use tinyjevclient::{Client, ClientConfig}; + use tinytools_jev::JevRanker; let client = if let Ok(key) = std::env::var("OPENHUMAN_BACKEND_API_KEY") { let mut client = ClientConfig::tinyhumans_openrouter(key); if let Ok(base) = std::env::var("BACKEND_URL") { @@ -287,24 +303,68 @@ fn jev_ranker(retrieval_k: usize) -> Option<(Arc, Arc, ranker)) } +#[cfg(feature = "jev")] +fn jev_config(retrieval_k: usize, family: bool, embedding: bool) -> tinytools_jev::JevRankerConfig { + use tinytools_jev::{JevRankerConfig, JevStrategy}; + let mut config = JevRankerConfig::new().with_retrieval_k(retrieval_k); + if family { + config = config.with_strategy(JevStrategy::FamilyThenDecide); + } + if embedding { + config = config.with_retriever(embedding_retriever()); + } + config +} + #[cfg(not(feature = "jev"))] -fn jev_ranker(_retrieval_k: usize) -> Option<(Arc, Arc<()>)> { +fn jev_ranker( + _retrieval_k: usize, + _family: bool, + _embedding: bool, +) -> Option<(Arc, Arc<()>)> { None } +/// The process's configured embedding provider as a `ToolRanker`, with its +/// catalogue cache in the scratch workspace so repeated runs embed only the +/// intents. +fn embedding_retriever() -> Arc { + use openhuman_core::agent::tinyagents::discovery::EmbeddingToolRanker; + let config = tokio::task::block_in_place(|| { + tokio::runtime::Handle::current().block_on(openhuman_core::config::Config::load_or_init()) + }) + .expect("load config for the embedding provider"); + let provider = + openhuman_core::inference::embedding_host::default_embedding_provider_with_config(&config); + if !EmbeddingToolRanker::provider_is_usable(provider.as_ref()) { + eprintln!( + "embedding: provider `{}` cannot embed; the bench retriever stays bm25", + provider.name() + ); + return Arc::new(Bm25Ranker); + } + eprintln!("embedding: {} / {}", provider.name(), provider.model_id()); + Arc::new( + EmbeddingToolRanker::new(provider).with_disk_cache( + repo_root().join("target").join("tool_search_bench_embeddings.json"), + ), + ) +} + fn family_of<'a>(catalogue: &'a [CatalogueEntry], name: &str) -> &'a str { catalogue .iter() @@ -358,19 +418,39 @@ async fn main() -> Result<()> { if want("overlap") { rankers.push(("overlap".into(), Arc::new(OverlapRanker))); } + if want("embedding") && args.embedding { + rankers.push(("embedding".into(), embedding_retriever())); + } if want("jev") { - match jev_ranker(args.retrieval_k) { + match jev_ranker(args.retrieval_k, args.family, args.embedding) { Some((ranker, _)) => rankers.push(("jev".into(), ranker)), - None => eprintln!( - "jev: skipped (set OPENHUMAN_BACKEND_API_KEY or TYPESAFE_API_KEY; build with the `jev` feature)" - ), + None => { + #[cfg(feature = "jev")] + { + let ranker = openhuman_tinyhumans::jev::TinyHumansJevRanker::with_config( + jev_config(args.retrieval_k, args.family, args.embedding), + ) + .with_deadline(Duration::from_secs(20)); + rankers.push(("jev".into(), Arc::new(ranker))); + } + #[cfg(not(feature = "jev"))] + eprintln!("jev: skipped (build with the `jev` feature)"); + } } } let mut reports = Vec::new(); for (kind, ranker) in &rankers { let mut report = RankerReport { - ranker: kind.clone(), + ranker: if kind == "jev" { + format!( + "jev({}{})", + if args.family { "family" } else { "retrieve" }, + if args.embedding { "+embedding" } else { "+bm25" } + ) + } else { + kind.clone() + }, rows: rows.len(), ..RankerReport::default() }; @@ -399,11 +479,25 @@ async fn main() -> Result<()> { continue; } report.labelled += 1; - if got.first().map(String::as_str) == Some(row.expected.as_str()) { + let source = if catalogue + .iter() + .any(|e| e.name == row.expected && e.family.as_deref().is_some_and(|f| FIXTURE_TOOLKITS.contains(&f))) + { + "composio" + } else { + "core" + }; + let bucket = report.by_source.entry(source.to_string()).or_default(); + bucket.0 += 1; + let hit1 = got.first().map(String::as_str) == Some(row.expected.as_str()); + let hit3 = got.iter().any(|g| g == &row.expected); + if hit1 { report.top1 += 1; + bucket.1 += 1; } - if got.iter().any(|g| g == &row.expected) { + if hit3 { report.top3 += 1; + bucket.2 += 1; } else if args.misses { report.misses.push(Miss { intent: row.intent.clone(), @@ -411,9 +505,31 @@ async fn main() -> Result<()> { got: got.clone(), }); } - let retrieved = Bm25Ranker::rank_sync(&candidates, &row.intent, args.retrieval_k); - if retrieved.iter().any(|h| h.key == row.expected) { + // Recall of the retriever Jev sits on: the embedding index when + // `--embedding`, BM25 otherwise. For the standalone rankers it is + // their own recall at `retrieval_k`. + let retrieved: Vec = if kind == "jev" && !args.embedding || kind == "bm25" { + Bm25Ranker::rank_sync(&candidates, &row.intent, args.retrieval_k) + .into_iter() + .map(|h| h.key) + .collect() + } else { + let retriever: Arc = if kind == "jev" { + embedding_retriever() + } else { + ranker.clone() + }; + retriever + .rank(&row.intent, &RankContext::empty(), &candidates, args.retrieval_k) + .await + .map(|hits| hits.into_iter().map(|h| h.key).collect()) + .unwrap_or_default() + }; + if retrieved.iter().any(|h| h == &row.expected) { report.recall_at_20 += 1; + if let Some(bucket) = report.by_source.get_mut(source) { + bucket.3 += 1; + } } let expected_family = row .family @@ -437,7 +553,7 @@ async fn main() -> Result<()> { if let Some(report) = reports.iter_mut().find(|r| r.ranker == "jev") { // Tokens and cost: one detailed pass over the labelled rows so the // number is the provider's own `usage`, not an estimate. - if let Some((_, detailed)) = jev_ranker(args.retrieval_k) { + if let Some((_, detailed)) = jev_ranker(args.retrieval_k, args.family, args.embedding) { let mut tokens = 0_u64; let mut counted = 0_u64; for row in rows.iter().take(25) { @@ -459,7 +575,7 @@ async fn main() -> Result<()> { } } - println!("| ranker | rows | top-1 | top-3 | recall@{} (bm25) | needless (of {}) | errors | p50 ms | p95 ms | tokens/search | USD/search |", + println!("| ranker | rows | top-1 | top-3 | retriever recall@{} | needless (of {}) | errors | p50 ms | p95 ms | tokens/search | USD/search |", args.retrieval_k, reports.first().map_or(0, |r| r.none_rows)); println!("|---|---|---|---|---|---|---|---|---|---|---|"); @@ -486,6 +602,14 @@ async fn main() -> Result<()> { if r.usd == 0.0 { "-".to_string() } else { format!("${:.5}", r.usd) }, ); } + println!("\n| ranker | source | labelled | top-1 | top-3 | recall@{} |", args.retrieval_k); + println!("|---|---|---|---|---|---|"); + for r in &reports { + for (source, (n, t1, t3, rk)) in &r.by_source { + let pct = |x: usize| if *n == 0 { "n/a".to_string() } else { format!("{:.1}%", 100.0 * x as f64 / *n as f64) }; + println!("| {} | {} | {} | {} | {} | {} |", r.ranker, source, n, pct(*t1), pct(*t3), pct(*rk)); + } + } for r in &reports { println!("\n### {} — top-1 family confusion (expected → got)", r.ranker); for (expected, gots) in &r.confusion { diff --git a/crates/openhuman-core/src/agent/tinyagents/discovery_tests.rs b/crates/openhuman-core/src/agent/tinyagents/discovery/discovery_tests.rs similarity index 100% rename from crates/openhuman-core/src/agent/tinyagents/discovery_tests.rs rename to crates/openhuman-core/src/agent/tinyagents/discovery/discovery_tests.rs diff --git a/crates/openhuman-core/src/agent/tinyagents/discovery/embedding_ranker.rs b/crates/openhuman-core/src/agent/tinyagents/discovery/embedding_ranker.rs new file mode 100644 index 0000000000..5ba8c1f4cf --- /dev/null +++ b/crates/openhuman-core/src/agent/tinyagents/discovery/embedding_ranker.rs @@ -0,0 +1,258 @@ +//! [`EmbeddingToolRanker`]: the semantic retriever for `tool_search`. +//! +//! A lexical retriever loses every paraphrase — "ping alex" never overlaps +//! `SLACK_SEND_MESSAGE send a message` — and a decision model can only pick +//! from what the retriever hands it. Measured on the orchestrator's catalogue +//! plus nine Composio toolkits (1,215 tools), BM25 recall@20 was 70%, and +//! that ceiling capped Jev at 67% top-3. This ranker embeds each tool's +//! summary once with the process's configured embedding provider (the same +//! one memory recall uses), embeds the intent per search, and ranks by cosine +//! similarity, so recall follows meaning rather than shared words. It is the +//! `retriever` inside `tinytools_jev::JevRanker` for every catalogue larger +//! than one Jev `Choice`. +//! +//! Catalogue embeddings are cached in memory by content hash and, when a +//! cache path is given, on disk keyed by the provider's signature, so a cold +//! process pays for the catalogue once and every later search embeds only +//! the intent (one provider call). + +use std::collections::HashMap; +use std::hash::{Hash, Hasher}; +use std::path::PathBuf; +use std::sync::{Arc, RwLock}; + +use serde::{Deserialize, Serialize}; +use tinytools::{RankCandidate, RankContext, RankError, RankHit, ToolRanker}; + +use crate::inference::embedding_host::EmbeddingProvider; + +/// Texts per embedding request. The managed provider accepts far more, but a +/// bounded batch keeps one request under any body cap and lets a partial +/// failure cost one batch, not the catalogue. +const EMBED_BATCH: usize = 64; + +/// Ranks tools by cosine similarity between the intent and each tool's +/// summary, with catalogue embeddings cached. +pub struct EmbeddingToolRanker { + provider: Arc, + cache: RwLock>>, + cache_path: Option, +} + +#[derive(Serialize, Deserialize, Default)] +struct DiskCache { + signature: String, + entries: HashMap>, +} + +impl std::fmt::Debug for EmbeddingToolRanker { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("EmbeddingToolRanker") + .field("provider", &self.provider.name()) + .field("model", &self.provider.model_id()) + .field("cache_path", &self.cache_path) + .finish_non_exhaustive() + } +} + +impl EmbeddingToolRanker { + /// The stable [`ToolRanker::kind`] of this ranker. + pub const KIND: &'static str = "embedding"; + + /// A ranker over `provider` with an in-memory cache only. + pub fn new(provider: Arc) -> Self { + Self { + provider, + cache: RwLock::new(HashMap::new()), + cache_path: None, + } + } + + /// A ranker whose catalogue embeddings also persist at `path`, keyed by + /// the provider's signature so a model change invalidates them. A + /// missing or unreadable file is an empty cache, never an error. + pub fn with_disk_cache(mut self, path: PathBuf) -> Self { + let loaded = std::fs::read(&path) + .ok() + .and_then(|raw| serde_json::from_slice::(&raw).ok()) + .filter(|disk| disk.signature == self.provider.signature()); + if let Some(disk) = loaded { + tracing::debug!( + entries = disk.entries.len(), + path = %path.display(), + "[tool-search] loaded embedding cache" + ); + *self.cache.write().unwrap_or_else(|p| p.into_inner()) = disk.entries; + } + self.cache_path = Some(path); + self + } + + /// Whether `provider` can embed at all. The `none` provider embeds + /// nothing and would rank everything at zero. + pub fn provider_is_usable(provider: &dyn EmbeddingProvider) -> bool { + provider.dimensions() > 0 && provider.name() != "none" + } + + fn key(candidate: &RankCandidate) -> u64 { + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + candidate.summary.hash(&mut hasher); + candidate.family.hash(&mut hasher); + hasher.finish() + } + + fn text(candidate: &RankCandidate) -> String { + match &candidate.family { + Some(family) => format!("{} ({family})", candidate.summary), + None => candidate.summary.clone(), + } + } + + /// Embeds every candidate not already cached, in batches. + async fn ensure_cached(&self, candidates: &[RankCandidate]) -> Result<(), RankError> { + let missing: Vec<(u64, String)> = { + let cache = self.cache.read().unwrap_or_else(|p| p.into_inner()); + let mut seen = std::collections::HashSet::new(); + candidates + .iter() + .map(|c| (Self::key(c), c)) + .filter(|(k, _)| !cache.contains_key(k) && seen.insert(*k)) + .map(|(k, c)| (k, Self::text(c))) + .collect() + }; + if missing.is_empty() { + return Ok(()); + } + tracing::info!( + missing = missing.len(), + provider = self.provider.name(), + "[tool-search] embedding catalogue entries" + ); + for batch in missing.chunks(EMBED_BATCH) { + let texts: Vec<&str> = batch.iter().map(|(_, t)| t.as_str()).collect(); + let vectors = self + .provider + .embed(&texts) + .await + .map_err(|error| RankError::Backend { + reason: format!("embedding failed: {error:#}"), + })?; + if vectors.len() != batch.len() { + return Err(RankError::Backend { + reason: format!( + "embedding returned {} vectors for {} texts", + vectors.len(), + batch.len() + ), + }); + } + let mut cache = self.cache.write().unwrap_or_else(|p| p.into_inner()); + for ((key, _), vector) in batch.iter().zip(vectors) { + cache.insert(*key, vector); + } + } + self.persist(); + Ok(()) + } + + fn persist(&self) { + let Some(path) = &self.cache_path else { + return; + }; + let disk = DiskCache { + signature: self.provider.signature(), + entries: self + .cache + .read() + .unwrap_or_else(|p| p.into_inner()) + .clone(), + }; + let write = || -> std::io::Result<()> { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent)?; + } + let tmp = path.with_extension("json.tmp"); + std::fs::write(&tmp, serde_json::to_vec(&disk)?)?; + std::fs::rename(&tmp, path) + }; + if let Err(error) = write() { + tracing::warn!( + path = %path.display(), + error = %error, + "[tool-search] could not persist embedding cache" + ); + } + } +} + +fn cosine(a: &[f32], b: &[f32]) -> f64 { + let (mut dot, mut na, mut nb) = (0.0_f64, 0.0_f64, 0.0_f64); + for (x, y) in a.iter().zip(b) { + dot += f64::from(*x) * f64::from(*y); + na += f64::from(*x) * f64::from(*x); + nb += f64::from(*y) * f64::from(*y); + } + if na == 0.0 || nb == 0.0 { + return 0.0; + } + dot / (na.sqrt() * nb.sqrt()) +} + +#[async_trait::async_trait] +impl ToolRanker for EmbeddingToolRanker { + fn kind(&self) -> &'static str { + Self::KIND + } + + async fn rank( + &self, + intent: &str, + _context: &RankContext, + candidates: &[RankCandidate], + limit: usize, + ) -> Result, RankError> { + let intent = intent.trim(); + if intent.is_empty() { + return Err(RankError::InvalidInput { + reason: "intent is empty".to_owned(), + }); + } + if candidates.is_empty() || limit == 0 { + return Ok(Vec::new()); + } + self.ensure_cached(candidates).await?; + let query = self + .provider + .embed(&[intent]) + .await + .map_err(|error| RankError::Backend { + reason: format!("embedding failed: {error:#}"), + })? + .into_iter() + .next() + .ok_or_else(|| RankError::Backend { + reason: "embedding returned no vector for the intent".to_owned(), + })?; + let cache = self.cache.read().unwrap_or_else(|p| p.into_inner()); + let mut scored: Vec = candidates + .iter() + .filter_map(|c| { + cache + .get(&Self::key(c)) + .map(|v| RankHit::new(c.key.clone(), cosine(&query, v))) + }) + .collect(); + scored.sort_by(|a, b| { + b.score + .partial_cmp(&a.score) + .unwrap_or(std::cmp::Ordering::Equal) + .then_with(|| a.key.cmp(&b.key)) + }); + scored.truncate(limit); + Ok(scored) + } +} + +#[cfg(test)] +#[path = "embedding_ranker_tests.rs"] +mod tests; diff --git a/crates/openhuman-core/src/agent/tinyagents/discovery/embedding_ranker_tests.rs b/crates/openhuman-core/src/agent/tinyagents/discovery/embedding_ranker_tests.rs new file mode 100644 index 0000000000..d0ab881fdd --- /dev/null +++ b/crates/openhuman-core/src/agent/tinyagents/discovery/embedding_ranker_tests.rs @@ -0,0 +1,151 @@ +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; + +use tinytools::{RankCandidate, RankContext, ToolRanker}; + +use super::*; + +/// Embeds a text as a bag of three hand-picked words, so similarity is +/// deterministic and readable. +struct BagEmbedder { + calls: AtomicUsize, +} + +#[async_trait::async_trait] +impl EmbeddingProvider for BagEmbedder { + fn name(&self) -> &str { + "bag" + } + fn model_id(&self) -> &str { + "bag-v1" + } + fn dimensions(&self) -> usize { + 3 + } + fn signature(&self) -> String { + "provider=bag;model=bag-v1;dims=3".into() + } + async fn embed(&self, texts: &[&str]) -> anyhow::Result>> { + self.calls.fetch_add(1, Ordering::SeqCst); + Ok(texts + .iter() + .map(|t| { + let t = t.to_ascii_lowercase(); + vec![ + f32::from(u8::from(t.contains("message") || t.contains("ping"))), + f32::from(u8::from(t.contains("email") || t.contains("mail"))), + f32::from(u8::from(t.contains("file"))), + ] + }) + .collect()) + } +} + +fn candidates() -> Vec { + vec![ + RankCandidate::new("SLACK_SEND_MESSAGE", "send a message to a channel").with_family("slack"), + RankCandidate::new("GMAIL_SEND_EMAIL", "send an email").with_family("gmail"), + RankCandidate::new("file_read", "read a file"), + ] +} + +#[tokio::test] +async fn ranks_by_cosine_and_embeds_the_catalogue_once() { + let embedder = Arc::new(BagEmbedder { + calls: AtomicUsize::new(0), + }); + let ranker = EmbeddingToolRanker::new(embedder.clone()); + assert_eq!(ranker.kind(), "embedding"); + + let hits = ranker + .rank("ping alex", &RankContext::empty(), &candidates(), 2) + .await + .unwrap(); + assert_eq!(hits[0].key, "SLACK_SEND_MESSAGE"); + assert!(hits[0].confidence.is_none()); + assert_eq!(hits.len(), 2); + // One batch for the catalogue plus one for the intent. + assert_eq!(embedder.calls.load(Ordering::SeqCst), 2); + + let hits = ranker + .rank("mail the report", &RankContext::empty(), &candidates(), 1) + .await + .unwrap(); + assert_eq!(hits[0].key, "GMAIL_SEND_EMAIL"); + // Only the intent was embedded this time. + assert_eq!(embedder.calls.load(Ordering::SeqCst), 3); +} + +#[tokio::test] +async fn disk_cache_round_trips_and_is_keyed_by_signature() { + let tmp = tempfile::tempdir().unwrap(); + let path = tmp.path().join("cache").join("tool_search_embeddings.json"); + let embedder = Arc::new(BagEmbedder { + calls: AtomicUsize::new(0), + }); + let ranker = EmbeddingToolRanker::new(embedder.clone()).with_disk_cache(path.clone()); + ranker + .rank("ping", &RankContext::empty(), &candidates(), 1) + .await + .unwrap(); + assert!(path.exists()); + + let embedder2 = Arc::new(BagEmbedder { + calls: AtomicUsize::new(0), + }); + let warm = EmbeddingToolRanker::new(embedder2.clone()).with_disk_cache(path.clone()); + warm.rank("ping", &RankContext::empty(), &candidates(), 1) + .await + .unwrap(); + assert_eq!( + embedder2.calls.load(Ordering::SeqCst), + 1, + "a warm cache embeds only the intent" + ); +} + +#[tokio::test] +async fn empty_intent_is_rejected_and_none_provider_is_unusable() { + let ranker = EmbeddingToolRanker::new(Arc::new(BagEmbedder { + calls: AtomicUsize::new(0), + })); + assert!(ranker + .rank(" ", &RankContext::empty(), &candidates(), 1) + .await + .is_err()); + let none = crate::inference::embedding_host::TinyInferenceEmbeddingProvider::new( + tinyinference_embeddings::NoopEmbeddingModel, + ); + assert!(!EmbeddingToolRanker::provider_is_usable(&none)); +} + +/// A tool that appears later — a newly connected toolkit's actions, a +/// rewritten description — is embedded on its own; the rest is a cache hit. +#[tokio::test] +async fn a_new_or_changed_tool_is_embedded_incrementally() { + let embedder = Arc::new(BagEmbedder { + calls: AtomicUsize::new(0), + }); + let ranker = EmbeddingToolRanker::new(embedder.clone()); + ranker + .rank("ping", &RankContext::empty(), &candidates(), 1) + .await + .unwrap(); + assert_eq!(embedder.calls.load(Ordering::SeqCst), 2, "catalogue + intent"); + + let mut grown = candidates(); + grown.push(RankCandidate::new("NOTION_CREATE_PAGE", "create a page").with_family("notion")); + grown[2] = RankCandidate::new("file_read", "read a file from disk"); + ranker + .rank("ping", &RankContext::empty(), &grown, 1) + .await + .unwrap(); + // One batch for the two unseen texts (the new tool and the changed one), + // plus the intent — never the whole catalogue again. + assert_eq!(embedder.calls.load(Ordering::SeqCst), 4); + assert_eq!( + ranker.cache.read().unwrap().len(), + 5, + "old and new descriptions both cached; a stale entry is harmless" + ); +} diff --git a/crates/openhuman-core/src/agent/tinyagents/discovery.rs b/crates/openhuman-core/src/agent/tinyagents/discovery/mod.rs similarity index 98% rename from crates/openhuman-core/src/agent/tinyagents/discovery.rs rename to crates/openhuman-core/src/agent/tinyagents/discovery/mod.rs index 92807ddbb7..f77dafa80d 100644 --- a/crates/openhuman-core/src/agent/tinyagents/discovery.rs +++ b/crates/openhuman-core/src/agent/tinyagents/discovery/mod.rs @@ -168,6 +168,10 @@ impl ToolRanker for OverlapRanker { } } +mod embedding_ranker; + +pub use embedding_ranker::EmbeddingToolRanker; + #[cfg(test)] #[path = "discovery_tests.rs"] mod tests; diff --git a/crates/openhuman-tinyhumans/Cargo.toml b/crates/openhuman-tinyhumans/Cargo.toml index dc7372aae7..081e79cbda 100644 --- a/crates/openhuman-tinyhumans/Cargo.toml +++ b/crates/openhuman-tinyhumans/Cargo.toml @@ -17,7 +17,7 @@ publish = false # default and forwarded by every shipped host; without it the core ranks # `tool_search` with BM25 alone. default = ["openhuman-embed/default", "jev"] -jev = ["dep:tinytools-jev"] +jev = ["dep:tinytools-jev", "dep:tinyjevclient"] http-server = ["openhuman-embed/http-server"] inference = ["openhuman-embed/inference"] documents = ["openhuman-embed/documents"] @@ -56,6 +56,10 @@ tinyhumans-sdk = { path = "../../vendor/tinyhumans-sdk", default-features = fals # `tinytools`, so the `ToolRanker` types unify. tinytools = { path = "../../vendor/tinyagents/vendor/tinytools/crates/tinytools" } tinytools-jev = { path = "../../vendor/tinyagents/vendor/tinytools/crates/tinytools-jev", optional = true } +# The System One transport behind the Jev evaluator; `tinytools-jev` is +# transport-free and this crate owns the wire, the credential and the deadline. +# Consumed by pinned revision, as that crate's README asks. +tinyjevclient = { git = "https://github.com/tinyhumansai/tinyjevclient", rev = "84b3983c7e1e14f7515658ceafabb6c4e7967c94", optional = true } anyhow = "1" async-trait = "0.1" chrono = { version = "0.4", default-features = false, features = ["clock", "std"] } diff --git a/crates/openhuman-tinyhumans/src/jev/evaluator.rs b/crates/openhuman-tinyhumans/src/jev/evaluator.rs new file mode 100644 index 0000000000..c0e3840812 --- /dev/null +++ b/crates/openhuman-tinyhumans/src/jev/evaluator.rs @@ -0,0 +1,160 @@ +//! [`TinyJevEvaluator`]: the `tinytools_jev::JevEvaluator` over +//! `tinyjevclient`, reaching Jev through the TinyHumans System One proxy. +//! +//! `tinytools-jev` decides *what* to ask — the options, the intent, the +//! family-stage wording — and this evaluator owns the wire: one `Choice` +//! over the options plus a `needs_tool` `Noul`, the credential, the +//! per-attempt timeout and retries the client applies, and a deadline of +//! its own so a slow answer becomes a fallback rather than a stalled turn. + +use std::{collections::BTreeMap, time::Duration}; + +use serde_json::{json, Value}; +use tinyjevclient::{ + Answer, Choice, Client, Error as JevError, EvaluationFailure, EvaluationRequest, Noul, + NoulCriteria, Question, +}; +use tinytools::RankError; +use tinytools_jev::{JevDecision, JevEvaluator, JevRequest}; + +/// Question id of the tool `Choice`. +const TOOL_QUESTION: &str = "tool"; +/// Question id of the needs-a-tool `Noul`. +const NEEDS_TOOL_QUESTION: &str = "needs_tool"; +/// Default deadline for one evaluation, on top of the client's own +/// per-attempt timeout and retries. +const DEFAULT_DEADLINE: Duration = Duration::from_secs(3); + +/// Evaluates `tinytools_jev` requests against Jev through `tinyjevclient`. +#[derive(Debug, Clone)] +pub struct TinyJevEvaluator { + client: Client, + deadline: Duration, +} + +impl TinyJevEvaluator { + /// An evaluator over `client` with the default 3 s deadline. + pub fn new(client: Client) -> Self { + Self { + client, + deadline: DEFAULT_DEADLINE, + } + } + + /// Sets the per-evaluation deadline. + pub fn with_deadline(mut self, deadline: Duration) -> Self { + self.deadline = deadline; + self + } + + fn build(request: &JevRequest) -> EvaluationRequest { + let mut state = json!({ "request": request.intent }); + if !request.recent_turns.is_empty() { + if let Some(object) = state.as_object_mut() { + object.insert( + "recent_user_turns".to_owned(), + Value::Array( + request + .recent_turns + .iter() + .map(|turn| Value::String(turn.clone())) + .collect(), + ), + ); + } + } + let criteria: BTreeMap> = request + .options + .iter() + .map(|option| (option.key.clone(), Some(json!(option.description)))) + .collect(); + let instructions = request.instructions.clone().unwrap_or_else(|| { + "Which tool accomplishes the user's `request`? Judge by what each tool does, \ + not by shared words. Pick `none` when no listed tool does it." + .to_owned() + }); + EvaluationRequest { + state, + model: request.model.clone(), + questions: BTreeMap::from([ + ( + TOOL_QUESTION.to_owned(), + Question::Choice(Choice { + instructions: json!(instructions), + criteria, + }), + ), + ( + NEEDS_TOOL_QUESTION.to_owned(), + Question::Noul(Noul { + instructions: json!( + "Does fulfilling the user's `request` require calling a tool \ + — an action or a lookup outside the assistant's own knowledge?" + ), + criteria: Some(NoulCriteria { + r#true: json!( + "The request asks for an action or for information that \ + must be fetched." + ), + r#false: json!( + "The request can be answered by replying, with no tool." + ), + }), + }), + ), + ]), + } + } +} + +fn map_failure(failure: EvaluationFailure) -> RankError { + match failure.error { + JevError::InvalidRequest { reason } | JevError::InvalidConfig { reason } => { + RankError::invalid_input(reason) + } + JevError::Timeout => RankError::Timeout, + // `Display` on every variant is credential-free by the client's + // contract; the transport source is dropped, not printed. + other => RankError::backend(format!("{other} after {} attempt(s)", failure.attempts)), + } +} + +#[async_trait::async_trait] +impl JevEvaluator for TinyJevEvaluator { + async fn evaluate(&self, request: &JevRequest) -> Result { + let wire = Self::build(request); + let result = tokio::time::timeout(self.deadline, self.client.evaluate(&wire)) + .await + .map_err(|_elapsed| RankError::Timeout)? + .map_err(map_failure)?; + let Some(Answer::Choice(choice)) = result.response.answers.get(TOOL_QUESTION) else { + return Err(RankError::backend( + "response has no choice answer for `tool`", + )); + }; + let needs_tool = match result.response.answers.get(NEEDS_TOOL_QUESTION) { + Some(Answer::Noul(noul)) => Some(noul.noul), + _ => None, + }; + log::debug!( + "[tool-search] jev evaluated {} option(s) (confidence={:.2} needs_tool={:?} latency_ms={} attempts={} input_tokens={:?})", + request.options.len(), + choice.confidence, + needs_tool, + result.latency.as_millis(), + result.attempts, + result.response.usage.input_tokens, + ); + Ok(JevDecision { + probabilities: choice.probabilities.clone(), + choice_confidence: choice.confidence, + needs_tool, + input_tokens: result.response.usage.input_tokens, + attempts: result.attempts, + }) + } +} + +#[cfg(test)] +#[path = "evaluator_tests.rs"] +mod tests; diff --git a/crates/openhuman-tinyhumans/src/jev/evaluator_tests.rs b/crates/openhuman-tinyhumans/src/jev/evaluator_tests.rs new file mode 100644 index 0000000000..dded49826f --- /dev/null +++ b/crates/openhuman-tinyhumans/src/jev/evaluator_tests.rs @@ -0,0 +1,73 @@ +use tinytools_jev::JevOption; + +use super::*; + +fn request(instructions: Option<&str>) -> JevRequest { + JevRequest { + intent: "ping alex".into(), + recent_turns: vec!["earlier turn".into()], + options: vec![ + JevOption { + key: "SLACK_SEND_MESSAGE".into(), + description: "send a message (from slack)".into(), + }, + JevOption { + key: "none".into(), + description: "No listed tool accomplishes the request.".into(), + }, + ], + model: "jev-latest".into(), + instructions: instructions.map(str::to_owned), + } +} + +#[test] +fn builds_one_choice_and_one_noul_from_the_request() { + let wire = TinyJevEvaluator::build(&request(None)); + assert_eq!(wire.model, "jev-latest"); + assert_eq!(wire.state["request"], "ping alex"); + assert_eq!(wire.state["recent_user_turns"][0], "earlier turn"); + assert_eq!(wire.questions.len(), 2); + match wire.questions.get("tool") { + Some(Question::Choice(choice)) => { + assert_eq!(choice.criteria.len(), 2); + assert!(choice.instructions.as_str().unwrap().starts_with("Which tool")); + } + other => panic!("expected a choice, got {other:?}"), + } + assert!(matches!( + wire.questions.get("needs_tool"), + Some(Question::Noul(_)) + )); +} + +#[test] +fn request_instructions_replace_the_default_wording() { + let wire = TinyJevEvaluator::build(&request(Some("Which group of tools applies?"))); + match wire.questions.get("tool") { + Some(Question::Choice(choice)) => { + assert_eq!(choice.instructions, json!("Which group of tools applies?")); + } + other => panic!("expected a choice, got {other:?}"), + } +} + +#[test] +fn failures_map_without_leaking_the_key() { + let failure = EvaluationFailure { + error: JevError::Authentication, + attempts: 2, + latency: Duration::from_millis(5), + }; + let text = map_failure(failure).to_string(); + assert!(text.contains("authentication failed"), "{text}"); + assert!(text.contains("2 attempt(s)"), "{text}"); + assert!(matches!( + map_failure(EvaluationFailure { + error: JevError::Timeout, + attempts: 1, + latency: Duration::ZERO, + }), + RankError::Timeout + )); +} diff --git a/crates/openhuman-tinyhumans/src/jev/mod.rs b/crates/openhuman-tinyhumans/src/jev/mod.rs index 963df39f6b..aca893176b 100644 --- a/crates/openhuman-tinyhumans/src/jev/mod.rs +++ b/crates/openhuman-tinyhumans/src/jev/mod.rs @@ -15,8 +15,10 @@ //! out. The built client is cached by credential and base URL so a stable //! session does not rebuild an HTTP client on every search. +mod evaluator; mod ranker; +pub use evaluator::TinyJevEvaluator; pub use ranker::TinyHumansJevRanker; use std::sync::Arc; diff --git a/crates/openhuman-tinyhumans/src/jev/ranker.rs b/crates/openhuman-tinyhumans/src/jev/ranker.rs index 37a33d8bb5..4c020c5913 100644 --- a/crates/openhuman-tinyhumans/src/jev/ranker.rs +++ b/crates/openhuman-tinyhumans/src/jev/ranker.rs @@ -7,13 +7,17 @@ use std::{ sync::Mutex, }; -use std::{future::Future, pin::Pin, sync::Arc}; +use std::{future::Future, pin::Pin, sync::Arc, time::Duration}; +use openhuman_core::agent::tinyagents::discovery::EmbeddingToolRanker; use openhuman_core::api::config::effective_backend_api_url; use openhuman_core::config::Config; use openhuman_core::security::credentials::session_support::resolve_backend_credential; use tinytools::{RankCandidate, RankContext, RankError, RankHit, ToolRanker}; -use tinytools_jev::{ClientConfig, JevRanker, JevRankerConfig}; +use tinyjevclient::{Client, ClientConfig}; +use tinytools_jev::{JevRanker, JevRankerConfig, JevStrategy}; + +use super::evaluator::TinyJevEvaluator; /// How the ranker reads the config a search runs under. The default is the /// core's own read path (the embedder's config when one is bound, else the @@ -27,12 +31,24 @@ pub type ConfigLoader = Arc< pub struct TinyHumansJevRanker { config: JevRankerConfig, load_config: ConfigLoader, + /// Deadline for one evaluation. Measured through the TinyHumans proxy + /// (2026-09) one evaluation takes 0.7–1.9 s at p50 and the family + /// strategy runs its second-stage evaluations concurrently, so six + /// seconds bounds a slow search well above the norm while still turning + /// a stalled proxy into a BM25 fallback inside the turn. + deadline: Duration, cached: Mutex>, } +/// Default per-evaluation deadline; see `TinyHumansJevRanker::deadline`. +const DEFAULT_DEADLINE: Duration = Duration::from_secs(6); + struct Cached { fingerprint: u64, ranker: JevRanker, + /// The retriever inside `ranker`, kept so its catalogue embeddings + /// survive a credential change. + retriever: Arc, } impl std::fmt::Debug for TinyHumansJevRanker { @@ -50,10 +66,14 @@ impl Default for TinyHumansJevRanker { } impl TinyHumansJevRanker { - /// A ranker with `tinytools-jev`'s defaults: BM25 retrieval to 20, one - /// Jev decision, a 3 s deadline. + /// A ranker with the product defaults: the process's embedding provider + /// retrieves the top 20 tools by meaning, one Jev evaluation decides + /// (`RetrieveThenDecide`), 6 s deadline per evaluation. Without a usable + /// embedding provider the search does not run and the harness ranks + /// with BM25 alone — a lexical shortlist would cap Jev at BM25's recall, + /// which the bench measured at 70% on the Composio catalogue. pub fn new() -> Self { - Self::with_config(JevRankerConfig::new()) + Self::with_config(JevRankerConfig::new().with_strategy(JevStrategy::RetrieveThenDecide)) } /// A ranker with an explicit `tinytools-jev` configuration. @@ -63,10 +83,17 @@ impl TinyHumansJevRanker { load_config: Arc::new(|| { Box::pin(openhuman_core::config::ops::load_config_with_timeout()) }), + deadline: DEFAULT_DEADLINE, cached: Mutex::new(None), } } + /// Sets the per-evaluation deadline. + pub fn with_deadline(mut self, deadline: Duration) -> Self { + self.deadline = deadline; + self + } + /// Reads the config through `loader` instead of the core's read path. pub fn with_config_loader(mut self, loader: ConfigLoader) -> Self { self.load_config = loader; @@ -102,9 +129,24 @@ impl TinyHumansJevRanker { if let Some(entry) = cached.as_ref().filter(|entry| entry.fingerprint == fingerprint) { return Ok(entry.ranker.clone()); } - let mut client = ClientConfig::tinyhumans_openrouter(credential.into_secret()); - client.base_url = base_url.clone(); - let ranker = JevRanker::from_config(client, self.config.clone())?; + let mut client_config = ClientConfig::tinyhumans_openrouter(credential.into_secret()); + client_config.base_url = base_url.clone(); + let client = Client::new(client_config) + .map_err(|error| RankError::invalid_input(error.to_string()))?; + let evaluator: Arc = + Arc::new(TinyJevEvaluator::new(client).with_deadline(self.deadline)); + // The retriever is the process's embedding provider when it can + // embed (the same one memory recall uses), so a family larger than + // one Jev Choice is cut by meaning, not by shared words. Reused + // across rebuilds so the catalogue is embedded once per process. + let retriever: Arc = match cached.as_ref() { + Some(entry) => entry.retriever.clone(), + None => retriever_for(&config)?, + }; + let ranker = JevRanker::new( + evaluator, + self.config.clone().with_retriever(retriever.clone()), + ); log::info!( "[tool-search] jev ranker bound to backend {} ({})", openhuman_core::util::redact::redact_url_for_log(&base_url), @@ -117,11 +159,47 @@ impl TinyHumansJevRanker { *cached = Some(Cached { fingerprint, ranker: ranker.clone(), + retriever, }); Ok(ranker) } } +/// The semantic retriever for `config`'s embedding provider. +/// +/// A provider that cannot embed (`none`) is an error, not a BM25 substitute: +/// the harness answers the search with its own BM25 catalogue in that case, +/// and a Jev decision over a lexical shortlist would only add a network +/// round trip to the same recall. +fn retriever_for(config: &Config) -> Result, RankError> { + let provider = openhuman_core::inference::embedding_host::default_embedding_provider_with_config( + config, + ); + if !EmbeddingToolRanker::provider_is_usable(provider.as_ref()) { + log::info!( + "[tool-search] embedding provider `{}` cannot embed; jev search disabled, bm25 answers", + provider.name() + ); + return Err(RankError::backend(format!( + "no usable embedding provider (`{}`); jev search disabled", + provider.name() + ))); + } + log::info!( + "[tool-search] retrieving with embeddings ({} / {})", + provider.name(), + provider.model_id() + ); + Ok(Arc::new( + EmbeddingToolRanker::new(provider).with_disk_cache( + config + .workspace_dir + .join("cache") + .join("tool_search_embeddings.json"), + ), + )) +} + fn fingerprint(secret: &str, base_url: &str) -> u64 { let mut hasher = DefaultHasher::new(); secret.hash(&mut hasher); @@ -151,9 +229,10 @@ impl ToolRanker for TinyHumansJevRanker { .rank_detailed(intent, context, candidates, limit) .await?; log::debug!( - "[tool-search] jev ranked {} of {} shortlisted (choice_confidence={:.2} needs_tool={:?} none={:.2} latency_ms={} attempts={} input_tokens={:?})", + "[tool-search] jev ranked {} of {} shown (families={:?} choice_confidence={:.2} needs_tool={:?} none={:.2} latency_ms={} attempts={} input_tokens={:?})", ranking.hits.len(), ranking.shortlisted, + ranking.families, ranking.choice_confidence, ranking.needs_tool, ranking.none_probability, diff --git a/crates/openhuman-tinyhumans/src/jev/ranker_tests.rs b/crates/openhuman-tinyhumans/src/jev/ranker_tests.rs index b3b7501e57..d91995a832 100644 --- a/crates/openhuman-tinyhumans/src/jev/ranker_tests.rs +++ b/crates/openhuman-tinyhumans/src/jev/ranker_tests.rs @@ -51,3 +51,24 @@ async fn no_credential_is_a_backend_error_naming_the_gap() { other => panic!("expected a backend error, got {other}"), } } + +/// A config whose embedding provider is `none` disables the Jev search +/// outright — the harness's BM25 answers — rather than quietly retrieving +/// lexically. +#[test] +fn no_embedding_provider_disables_the_search() { + let tmp = tempfile::tempdir().expect("tempdir"); + let mut config = Config { + workspace_dir: tmp.path().join("workspace"), + action_dir: tmp.path().join("workspace"), + config_path: tmp.path().join("config.toml"), + ..Config::default() + }; + config.memory.embedding_provider = "none".into(); + let err = retriever_for(&config).err().map(|e| e.to_string()); + assert!( + err.as_deref() + .is_some_and(|e| e.contains("no usable embedding provider")), + "{err:?}" + ); +} diff --git a/docs/plans/jev-tool-search-baseline.md b/docs/plans/jev-tool-search-baseline.md new file mode 100644 index 0000000000..578d1bb23b --- /dev/null +++ b/docs/plans/jev-tool-search-baseline.md @@ -0,0 +1,96 @@ +# `tool_search` ranking: baseline and results + +Measured 2026-09-22 with `cargo run -p openhuman-cli --bin tool-search-bench`, +live Jev (`jev-1.13` through the TinyHumans System One proxy, signed-in +session) and the managed `embedding-v1` embedder. Catalogue: every tool the +orchestrator session registers (215) plus the recorded Composio catalogues +under `tests/fixtures/composio_*.json` (1,000 actions across gmail, slack, +github, notion, googledrive, googlesheets, reddit, facebook, instagram) as the +deferred per-action tools a signed-in workspace synthesises. Intents: +`tests/fixtures/tool_search/intents.jsonl` — 160 hand-written requests, 66 +labelled with a Composio action, 63 with a core tool, 31 that no tool should +answer. + +Before this work the orchestrator reached a Composio action only through +`delegate_to_integrations_agent` → an `integrations_agent` sub-run whose +toolkit was narrowed by `rank_tools_by_prompt` (the `overlap` row). Every +`tool_search` row is one search followed by a direct call of the tool it +returns; no sub-agent. + +## Rankers + +| ranker | rows | top-1 | top-3 | retriever recall@20 | needless (of 31) | errors | p50 ms | p95 ms | +|---|---|---|---|---|---|---|---|---| +| bm25 | 160 | 22.5% | 38.0% | 70.5% | 26 | 0 | 28 | 29 | +| overlap (`rank_tools_by_prompt`, the sub-agent's narrowing today) | 160 | 35.7% | 52.7% | 69.0% | 27 | 0 | 25 | 27 | +| Jev, BM25 top-20 then decide | 160 | 57.4% | 62.0% | 70.5% | 1 | 21† | 1542 | 3598 | +| Jev, embedding top-20 then decide (**product default**) | 160 | 62.0% | 66.7% | 86.8% | 1 | 5 | 1527 | 2611 | +| Jev only, family then decide (BM25 cut for >254) | 160 | 62.0–64.3% | 67.4–69.0% | 70.5% | 1 | 0–2 | 1275 | 2138 | +| Jev, family then decide, embedding cut for >254 | 160 | 62.8% | 67.4% | 86.8% | 1 | 4 | 1287 | 2018 | + +† the 3 s per-evaluation deadline of an earlier build; raised to 6 s in the +product and 20 s in the bench, after which errors are the residual proxy +timeouts shown on the other rows. + +## Composio actions — the heavy catalogue + +| ranker | labelled | top-1 | top-3 | retriever recall@20 | +|---|---|---|---|---| +| bm25 | 66 | 18.2% | 36.4% | 72.7% | +| overlap | 66 | 34.8% | 50.0% | 68.2% | +| Jev, BM25 top-20 | 66 | 66.7% | 72.7% | 72.7% | +| Jev, embedding top-20 | 66 | 74.2% | 78.8% | 90.9% | +| Jev only, family then decide | 66 | 77.3–83.3% | 86.4–90.9% | 72.7% | +| Jev, family then decide + embedding cut | 66 | 80.3% | 87.9% | 90.9% | + +Ranges are two runs of the same configuration: Jev's answers vary by a few +points run to run. + +What the rows say: + +- **Retrieval was the ceiling.** With BM25 shortlisting, Jev's Composio top-3 + (72.7%) equals BM25's recall@20 (72.7%): Jev picked correctly from + everything it was shown. A paraphrase ("ping alex" → `SLACK_SEND_MESSAGE`) + never reached it. +- **Letting Jev pick the family first removes the shortlist for every toolkit + that fits one choice** (all but GitHub's 500 actions), and Composio top-3 + goes to 86–91%. The remaining misses are near-synonyms + (`NOTION_APPEND_TEXT_BLOCKS` for `NOTION_ADD_PAGE_CONTENT`, + `INSTAGRAM_GET_IG_MEDIA_COMMENTS` for `INSTAGRAM_GET_POST_COMMENTS`) and + GitHub actions the BM25 cut dropped. +- **Embeddings lift recall@20 to 90.9%** on Composio, and that retriever + with one Jev decision is the product default: `RetrieveThenDecide` over + `EmbeddingToolRanker`, one proxy round trip. Family-then-decide scores a + few points higher on Composio at a second round trip and stays available + through `JevRankerConfig::with_strategy`. Catalogue embeddings are computed + once per process (19 batches of 64 for this catalogue) and cached on disk + under `/cache/tool_search_embeddings.json`, keyed by the + provider's signature; every later search embeds only the intent. +- **No embedder, no Jev search.** When the configured embedding provider is + `none`, `TinyHumansJevRanker` returns an error and the harness answers with + its own BM25 catalogue — a Jev decision over a lexical shortlist would only + add a round trip to the same recall. +- **Needless calls collapse**: 26/31 tool-less requests got a BM25 hit; every + Jev configuration answers at most one, because Jev's `none` option and + `needs_tool` abstain. +- **Core tools score lower under Jev than Composio** (46–54% top-3) because + `tinytools-jev` now abstains when `needs_tool < 0.5` or `none` beats the + best option, and many core-tool intents ("what did I tell you about my + dog?", "show me my todos") read as answerable without a tool. In the + product these tools are `Direct` — on the wire, never searched — so the + Composio column is the one `tool_search` is measured by. +- **Latency** is 1.3 s p50 for the family strategy (two proxy round trips, + the second stage's families evaluated concurrently), against a sub-agent + run of several model calls. + +## Reproducing + +```text +cargo run -p openhuman-cli --bin tool-search-bench -- --ranker all --misses +cargo run -p openhuman-cli --bin tool-search-bench -- --ranker jev --family --embedding +cargo run -p openhuman-cli --bin tool-search-bench -- --dump-catalogue +``` + +`OPENHUMAN_BACKEND_API_KEY` (or `TYPESAFE_API_KEY`) selects the key; without +one the bench ranks through the signed-in TinyHumans session exactly as the +product does. diff --git a/tests/fixtures/tool_search/intents.jsonl b/tests/fixtures/tool_search/intents.jsonl new file mode 100644 index 0000000000..f7ec573971 --- /dev/null +++ b/tests/fixtures/tool_search/intents.jsonl @@ -0,0 +1,162 @@ +# One JSON object per line: {intent, expected (tool name or "none"), family?}. +# Hand-written paraphrases over the real orchestrator registry and the recorded Composio catalogues; read by `tool-search-bench`. +{"intent": "ping alex by email that the deck is ready", "expected": "GMAIL_SEND_EMAIL"} +{"intent": "shoot a quick mail to the team about tomorrow's standup being cancelled", "expected": "GMAIL_SEND_EMAIL"} +{"intent": "what's new in my inbox this morning?", "expected": "GMAIL_FETCH_EMAILS"} +{"intent": "did anyone from Acme write to me this week?", "expected": "GMAIL_FETCH_EMAILS"} +{"intent": "draft a reply to Sarah's proposal but don't send it yet", "expected": "GMAIL_CREATE_EMAIL_DRAFT"} +{"intent": "answer that thread from the landlord saying yes", "expected": "GMAIL_REPLY_TO_THREAD"} +{"intent": "what labels do I have set up in gmail?", "expected": "GMAIL_LIST_LABELS"} +{"intent": "pull the PDF attached to the invoice email", "expected": "GMAIL_GET_ATTACHMENT"} +{"intent": "bin that spammy newsletter email", "expected": "GMAIL_MOVE_TO_TRASH"} +{"intent": "tag the flight confirmation with my Travel label", "expected": "GMAIL_ADD_LABEL_TO_EMAIL"} +{"intent": "look up Priya's email address from my contacts", "expected": "GMAIL_SEARCH_PEOPLE"} +{"intent": "ping alex on slack that I'm ten minutes late", "expected": "SLACK_SEND_MESSAGE"} +{"intent": "tell #eng in slack the deploy is done", "expected": "SLACK_SEND_MESSAGE"} +{"intent": "drop a \ud83d\udc4d on the last message in #general", "expected": "SLACK_ADD_REACTION_TO_AN_ITEM"} +{"intent": "what did people say in #design yesterday?", "expected": "SLACK_FETCH_CONVERSATION_HISTORY"} +{"intent": "find the slack thread where we discussed the pricing page", "expected": "SLACK_SEARCH_MESSAGES"} +{"intent": "make a new slack channel called launch-week", "expected": "SLACK_CREATE_CHANNEL"} +{"intent": "list all the channels in our workspace", "expected": "SLACK_LIST_ALL_CHANNELS"} +{"intent": "add maria to the #ops channel", "expected": "SLACK_INVITE_USER_TO_CHANNEL"} +{"intent": "schedule a slack message to #team for monday 9am saying happy new week", "expected": "SLACK_SCHEDULE_MESSAGE"} +{"intent": "pin that announcement in the channel", "expected": "SLACK_PIN_ITEM"} +{"intent": "post the roadmap PDF into #product on slack", "expected": "SLACK_UPLOAD_OR_CREATE_A_FILE_IN_SLACK"} +{"intent": "file a bug on the repo about the login page crash", "expected": "GITHUB_CREATE_AN_ISSUE"} +{"intent": "open a github issue: dark mode toggle is broken", "expected": "GITHUB_CREATE_AN_ISSUE"} +{"intent": "open a PR from fix/login into main", "expected": "GITHUB_CREATE_A_PULL_REQUEST"} +{"intent": "leave a comment on issue 42 saying we'll pick it up next sprint", "expected": "GITHUB_CREATE_AN_ISSUE_COMMENT"} +{"intent": "what PRs are open on the backend repo?", "expected": "GITHUB_FIND_PULL_REQUESTS"} +{"intent": "show me the recent commits on main", "expected": "GITHUB_LIST_COMMITS"} +{"intent": "cut a v1.2.0 release on github", "expected": "GITHUB_CREATE_A_RELEASE"} +{"intent": "label issue 17 as bug and p1", "expected": "GITHUB_ADD_LABELS_TO_AN_ISSUE"} +{"intent": "fork the tinytools repo into my account", "expected": "GITHUB_CREATE_A_FORK"} +{"intent": "has PR 88 been merged yet?", "expected": "GITHUB_CHECK_IF_PULL_REQUEST_HAS_BEEN_MERGED"} +{"intent": "get the readme of the openhuman repository", "expected": "GITHUB_GET_A_REPOSITORY_README"} +{"intent": "approve pull request 12 with a review", "expected": "GITHUB_CREATE_A_REVIEW_FOR_A_PULL_REQUEST"} +{"intent": "make a notion page for the offsite agenda", "expected": "NOTION_CREATE_NOTION_PAGE"} +{"intent": "find my notion page about hiring", "expected": "NOTION_SEARCH_NOTION_PAGE"} +{"intent": "append these meeting notes to the notion page", "expected": "NOTION_ADD_PAGE_CONTENT"} +{"intent": "set up a notion database to track candidates", "expected": "NOTION_CREATE_DATABASE"} +{"intent": "which rows in the notion tasks database are overdue?", "expected": "NOTION_QUERY_DATABASE_WITH_FILTER"} +{"intent": "rename the notion page to Q4 planning", "expected": "NOTION_UPDATE_PAGE"} +{"intent": "archive the old roadmap page in notion", "expected": "NOTION_ARCHIVE_NOTION_PAGE"} +{"intent": "leave a comment on the notion spec asking about scope", "expected": "NOTION_CREATE_COMMENT"} +{"intent": "upload the contract to my google drive", "expected": "GOOGLEDRIVE_UPLOAD_FILE"} +{"intent": "where's the budget spreadsheet in my drive?", "expected": "GOOGLEDRIVE_FIND_FILE"} +{"intent": "create a Receipts folder in drive", "expected": "GOOGLEDRIVE_CREATE_FOLDER"} +{"intent": "download the onboarding doc from google drive", "expected": "GOOGLEDRIVE_DOWNLOAD_FILE"} +{"intent": "share the pitch deck in drive with tom@example.com", "expected": "GOOGLEDRIVE_CREATE_PERMISSION"} +{"intent": "move the photos folder into Archive on drive", "expected": "GOOGLEDRIVE_MOVE_FILE"} +{"intent": "make a copy of the template doc in drive", "expected": "GOOGLEDRIVE_COPY_FILE"} +{"intent": "delete the duplicate file from google drive", "expected": "GOOGLEDRIVE_DELETE_FILE"} +{"intent": "read the values in A1:D20 of the sales sheet", "expected": "GOOGLESHEETS_BATCH_GET"} +{"intent": "add a new tab called July to the expenses spreadsheet", "expected": "GOOGLESHEETS_ADD_SHEET"} +{"intent": "append a row with today's numbers to the metrics sheet", "expected": "GOOGLESHEETS_SPREADSHEETS_VALUES_APPEND"} +{"intent": "start a fresh google sheet for the vendor list", "expected": "GOOGLESHEETS_CREATE_GOOGLE_SHEET1"} +{"intent": "wipe the values in the scratch range of the sheet", "expected": "GOOGLESHEETS_CLEAR_VALUES"} +{"intent": "replace every 'TBD' with 'done' in the tracker spreadsheet", "expected": "GOOGLESHEETS_FIND_REPLACE"} +{"intent": "post to r/rust about our new crate", "expected": "REDDIT_CREATE_REDDIT_POST"} +{"intent": "search reddit for threads about the M4 macbook", "expected": "REDDIT_SEARCH_ACROSS_SUBREDDITS"} +{"intent": "what are the comments on that reddit post?", "expected": "REDDIT_RETRIEVE_POST_COMMENTS"} +{"intent": "reply to the top comment on my reddit post", "expected": "REDDIT_POST_REDDIT_COMMENT"} +{"intent": "publish a post on our facebook page about the sale", "expected": "FACEBOOK_CREATE_POST"} +{"intent": "how are our facebook page posts performing?", "expected": "FACEBOOK_GET_PAGE_INSIGHTS"} +{"intent": "which facebook pages do I manage?", "expected": "FACEBOOK_LIST_MANAGED_PAGES"} +{"intent": "show me the comments on my latest instagram post", "expected": "INSTAGRAM_GET_POST_COMMENTS"} +{"intent": "how many followers and reach did my instagram get this week?", "expected": "INSTAGRAM_GET_USER_INSIGHTS"} +{"intent": "prepare an instagram post with this photo", "expected": "INSTAGRAM_CREATE_MEDIA_CONTAINER"} +{"intent": "open src/main.rs and show me the contents", "expected": "file_read"} +{"intent": "what's in the README?", "expected": "file_read"} +{"intent": "find every place we call parse_config", "expected": "grep"} +{"intent": "which files mention TODO in the crates folder?", "expected": "grep"} +{"intent": "list all the .toml files in the repo", "expected": "glob"} +{"intent": "what's in the current directory?", "expected": "list"} +{"intent": "write a hello world script to hello.py", "expected": "file_write"} +{"intent": "apply this diff to the parser", "expected": "apply_patch"} +{"intent": "run the test suite", "expected": "shell"} +{"intent": "execute npm install", "expected": "shell"} +{"intent": "commit these changes with the message fix typo", "expected": "git_operations"} +{"intent": "what's the latest news about the fed rate decision?", "expected": "web_search_tool"} +{"intent": "google who won the champions league", "expected": "web_search_tool"} +{"intent": "fetch https://example.com/pricing and summarize it", "expected": "web_fetch"} +{"intent": "call the weather API at api.weather.example/today", "expected": "http_request"} +{"intent": "remember that I prefer short answers", "expected": "save_preference"} +{"intent": "what did I tell you about my dog?", "expected": "memory_recall"} +{"intent": "store the fact that my flight is on the 14th", "expected": "memory_store"} +{"intent": "forget what I said about the old address", "expected": "memory_forget"} +{"intent": "what time is it in Tokyo right now?", "expected": "current_time"} +{"intent": "what's next friday's date?", "expected": "resolve_time"} +{"intent": "remind me every morning at 8 to drink water", "expected": "cron_add"} +{"intent": "what scheduled jobs do I have?", "expected": "cron_list"} +{"intent": "cancel the weekly report cron", "expected": "cron_remove"} +{"intent": "add buy milk to my todo list", "expected": "todo_add"} +{"intent": "show me my todos", "expected": "todo_list"} +{"intent": "draw a picture of a cat astronaut", "expected": "create_image"} +{"intent": "make a short video of waves at sunset", "expected": "create_video"} +{"intent": "what's in this screenshot?", "expected": "analyze_image"} +{"intent": "do a deep dive on competitors in the AI note-taking space", "expected": "research"} +{"intent": "write and run a python script that sums a list", "expected": "run_code"} +{"intent": "run this python snippet: print(2**10)", "expected": "python_exec"} +{"intent": "is there a skill for summarizing PDFs?", "expected": "skill_search"} +{"intent": "search the skill registry for a stripe integration", "expected": "skill_registry_search"} +{"intent": "install the skill from https://github.com/x/y", "expected": "install_workflow_from_url"} +{"intent": "what workflows have I saved?", "expected": "list_workflows"} +{"intent": "run the weekly digest workflow", "expected": "run_workflow"} +{"intent": "check if there's an app update", "expected": "update_check"} +{"intent": "update the app to the newest version", "expected": "update_apply"} +{"intent": "set a goal to ship v2 by december", "expected": "goal_set"} +{"intent": "what's my current goal?", "expected": "goal_get"} +{"intent": "is the app healthy? run diagnostics", "expected": "doctor_health"} +{"intent": "how much have I spent on AI this month?", "expected": "cost_get_summary"} +{"intent": "find an MCP server for postgres", "expected": "mcp_registry_search"} +{"intent": "call the query tool on the postgres mcp server", "expected": "mcp_registry_tool_call"} +{"intent": "connect my hubspot account", "expected": "oauth_connect_url"} +{"intent": "export these results as a csv", "expected": "csv_export"} +{"intent": "send a pushover notification to my phone", "expected": "pushover"} +{"intent": "unsubscribe me from all these marketing emails", "expected": "gmail_unsubscribe"} +{"intent": "swap 0.1 eth for usdc", "expected": "do_crypto"} +{"intent": "schedule the report to run tomorrow at noon", "expected": "schedule_task"} +{"intent": "how do I set up the proxy in openhuman?", "expected": "ask_docs"} +{"intent": "what does the persona file say about me?", "expected": "workspace_read_persona"} +{"intent": "what did we discuss in last month's emails about the merger?", "expected": "retrieve_memory"} +{"intent": "turn this into a reusable skill", "expected": "create_skill"} +{"intent": "are the background services running?", "expected": "service_status"} +{"intent": "run three research tasks in parallel on rust, go and zig", "expected": "spawn_parallel_agents"} +{"intent": "ask me which option I want before continuing", "expected": "ask_user_clarification"} +{"intent": "build a workflow that emails me the top HN posts daily", "expected": "build_workflow"} +{"intent": "suggest workflows I could automate", "expected": "suggest_workflows"} +{"intent": "pull my open tasks from linear", "expected": "task_source_list_tasks"} +{"intent": "what facets have you learned about me?", "expected": "learning_list_facets"} +{"intent": "list the artifacts from earlier today", "expected": "artifact_list"} +{"intent": "set my status to away", "expected": "none"} +{"intent": "hi!", "expected": "none"} +{"intent": "thanks, that's all", "expected": "none"} +{"intent": "what's the capital of australia?", "expected": "none"} +{"intent": "explain the difference between tcp and udp", "expected": "none"} +{"intent": "tell me a joke", "expected": "none"} +{"intent": "how are you today?", "expected": "none"} +{"intent": "what's 17 times 23?", "expected": "none"} +{"intent": "translate 'good morning' into spanish", "expected": "none"} +{"intent": "write a haiku about autumn", "expected": "none"} +{"intent": "who wrote pride and prejudice?", "expected": "none"} +{"intent": "summarize what you just said in one sentence", "expected": "none"} +{"intent": "what's a good name for a golden retriever?", "expected": "none"} +{"intent": "can you rephrase that more formally?", "expected": "none"} +{"intent": "why is the sky blue?", "expected": "none"} +{"intent": "give me three tips for better sleep", "expected": "none"} +{"intent": "ok", "expected": "none"} +{"intent": "what does HTTP stand for?", "expected": "none"} +{"intent": "is 97 a prime number?", "expected": "none"} +{"intent": "define the word 'ephemeral'", "expected": "none"} +{"intent": "what year did the berlin wall fall?", "expected": "none"} +{"intent": "help me think through whether to take the job offer", "expected": "none"} +{"intent": "write a limerick about rust borrow checker", "expected": "none"} +{"intent": "what's the plural of octopus?", "expected": "none"} +{"intent": "convert 5 miles to kilometers", "expected": "none"} +{"intent": "recommend a sci-fi novel", "expected": "none"} +{"intent": "sorry, ignore that last message", "expected": "none"} +{"intent": "what's your name?", "expected": "none"} +{"intent": "how many days are in a leap year?", "expected": "none"} +{"intent": "explain recursion like I'm five", "expected": "none"} +{"intent": "good night", "expected": "none"} diff --git a/vendor/tinyagents b/vendor/tinyagents index eefef72b30..8d421d8c9f 160000 --- a/vendor/tinyagents +++ b/vendor/tinyagents @@ -1 +1 @@ -Subproject commit eefef72b30c4d63cba4442db01c99c2940682302 +Subproject commit 8d421d8c9fddfc20b7776e9a8e82b49a0af4b07a