From a94f747bb60bf1c80755e96a8b91caa054f68a1d Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 22 Sep 2026 07:11:02 +0530 Subject: [PATCH 01/14] test: add tool search intent fixtures Add a JSONL fixture file containing intents for tool search tests, providing sample data to support test coverage for search functionality. Auto-committed-on: macbook --- tests/fixtures/tool_search/intents.jsonl | 162 +++++++++++++++++++++++ 1 file changed, 162 insertions(+) create mode 100644 tests/fixtures/tool_search/intents.jsonl diff --git a/tests/fixtures/tool_search/intents.jsonl b/tests/fixtures/tool_search/intents.jsonl new file mode 100644 index 0000000000..f7ec573971 --- /dev/null +++ b/tests/fixtures/tool_search/intents.jsonl @@ -0,0 +1,162 @@ +# One JSON object per line: {intent, expected (tool name or "none"), family?}. +# Hand-written paraphrases over the real orchestrator registry and the recorded Composio catalogues; read by `tool-search-bench`. +{"intent": "ping alex by email that the deck is ready", "expected": "GMAIL_SEND_EMAIL"} +{"intent": "shoot a quick mail to the team about tomorrow's standup being cancelled", "expected": "GMAIL_SEND_EMAIL"} +{"intent": "what's new in my inbox this morning?", "expected": "GMAIL_FETCH_EMAILS"} +{"intent": "did anyone from Acme write to me this week?", "expected": "GMAIL_FETCH_EMAILS"} +{"intent": "draft a reply to Sarah's proposal but don't send it yet", "expected": "GMAIL_CREATE_EMAIL_DRAFT"} +{"intent": "answer that thread from the landlord saying yes", "expected": "GMAIL_REPLY_TO_THREAD"} +{"intent": "what labels do I have set up in gmail?", "expected": "GMAIL_LIST_LABELS"} +{"intent": "pull the PDF attached to the invoice email", "expected": "GMAIL_GET_ATTACHMENT"} +{"intent": "bin that spammy newsletter email", "expected": "GMAIL_MOVE_TO_TRASH"} +{"intent": "tag the flight confirmation with my Travel label", "expected": "GMAIL_ADD_LABEL_TO_EMAIL"} +{"intent": "look up Priya's email address from my contacts", "expected": "GMAIL_SEARCH_PEOPLE"} +{"intent": "ping alex on slack that I'm ten minutes late", "expected": "SLACK_SEND_MESSAGE"} +{"intent": "tell #eng in slack the deploy is done", "expected": "SLACK_SEND_MESSAGE"} +{"intent": "drop a \ud83d\udc4d on the last message in #general", "expected": "SLACK_ADD_REACTION_TO_AN_ITEM"} +{"intent": "what did people say in #design yesterday?", "expected": "SLACK_FETCH_CONVERSATION_HISTORY"} +{"intent": "find the slack thread where we discussed the pricing page", "expected": "SLACK_SEARCH_MESSAGES"} +{"intent": "make a new slack channel called launch-week", "expected": "SLACK_CREATE_CHANNEL"} +{"intent": "list all the channels in our workspace", "expected": "SLACK_LIST_ALL_CHANNELS"} +{"intent": "add maria to the #ops channel", "expected": "SLACK_INVITE_USER_TO_CHANNEL"} +{"intent": "schedule a slack message to #team for monday 9am saying happy new week", "expected": "SLACK_SCHEDULE_MESSAGE"} +{"intent": "pin that announcement in the channel", "expected": "SLACK_PIN_ITEM"} +{"intent": "post the roadmap PDF into #product on slack", "expected": "SLACK_UPLOAD_OR_CREATE_A_FILE_IN_SLACK"} +{"intent": "file a bug on the repo about the login page crash", "expected": "GITHUB_CREATE_AN_ISSUE"} +{"intent": "open a github issue: dark mode toggle is broken", "expected": "GITHUB_CREATE_AN_ISSUE"} +{"intent": "open a PR from fix/login into main", "expected": "GITHUB_CREATE_A_PULL_REQUEST"} +{"intent": "leave a comment on issue 42 saying we'll pick it up next sprint", "expected": "GITHUB_CREATE_AN_ISSUE_COMMENT"} +{"intent": "what PRs are open on the backend repo?", "expected": "GITHUB_FIND_PULL_REQUESTS"} +{"intent": "show me the recent commits on main", "expected": "GITHUB_LIST_COMMITS"} +{"intent": "cut a v1.2.0 release on github", "expected": "GITHUB_CREATE_A_RELEASE"} +{"intent": "label issue 17 as bug and p1", "expected": "GITHUB_ADD_LABELS_TO_AN_ISSUE"} +{"intent": "fork the tinytools repo into my account", "expected": "GITHUB_CREATE_A_FORK"} +{"intent": "has PR 88 been merged yet?", "expected": "GITHUB_CHECK_IF_PULL_REQUEST_HAS_BEEN_MERGED"} +{"intent": "get the readme of the openhuman repository", "expected": "GITHUB_GET_A_REPOSITORY_README"} +{"intent": "approve pull request 12 with a review", "expected": "GITHUB_CREATE_A_REVIEW_FOR_A_PULL_REQUEST"} +{"intent": "make a notion page for the offsite agenda", "expected": "NOTION_CREATE_NOTION_PAGE"} +{"intent": "find my notion page about hiring", "expected": "NOTION_SEARCH_NOTION_PAGE"} +{"intent": "append these meeting notes to the notion page", "expected": "NOTION_ADD_PAGE_CONTENT"} +{"intent": "set up a notion database to track candidates", "expected": "NOTION_CREATE_DATABASE"} +{"intent": "which rows in the notion tasks database are overdue?", "expected": "NOTION_QUERY_DATABASE_WITH_FILTER"} +{"intent": "rename the notion page to Q4 planning", "expected": "NOTION_UPDATE_PAGE"} +{"intent": "archive the old roadmap page in notion", "expected": "NOTION_ARCHIVE_NOTION_PAGE"} +{"intent": "leave a comment on the notion spec asking about scope", "expected": "NOTION_CREATE_COMMENT"} +{"intent": "upload the contract to my google drive", "expected": "GOOGLEDRIVE_UPLOAD_FILE"} +{"intent": "where's the budget spreadsheet in my drive?", "expected": "GOOGLEDRIVE_FIND_FILE"} +{"intent": "create a Receipts folder in drive", "expected": "GOOGLEDRIVE_CREATE_FOLDER"} +{"intent": "download the onboarding doc from google drive", "expected": "GOOGLEDRIVE_DOWNLOAD_FILE"} +{"intent": "share the pitch deck in drive with tom@example.com", "expected": "GOOGLEDRIVE_CREATE_PERMISSION"} +{"intent": "move the photos folder into Archive on drive", "expected": "GOOGLEDRIVE_MOVE_FILE"} +{"intent": "make a copy of the template doc in drive", "expected": "GOOGLEDRIVE_COPY_FILE"} +{"intent": "delete the duplicate file from google drive", "expected": "GOOGLEDRIVE_DELETE_FILE"} +{"intent": "read the values in A1:D20 of the sales sheet", "expected": "GOOGLESHEETS_BATCH_GET"} +{"intent": "add a new tab called July to the expenses spreadsheet", "expected": "GOOGLESHEETS_ADD_SHEET"} +{"intent": "append a row with today's numbers to the metrics sheet", "expected": "GOOGLESHEETS_SPREADSHEETS_VALUES_APPEND"} +{"intent": "start a fresh google sheet for the vendor list", "expected": "GOOGLESHEETS_CREATE_GOOGLE_SHEET1"} +{"intent": "wipe the values in the scratch range of the sheet", "expected": "GOOGLESHEETS_CLEAR_VALUES"} +{"intent": "replace every 'TBD' with 'done' in the tracker spreadsheet", "expected": "GOOGLESHEETS_FIND_REPLACE"} +{"intent": "post to r/rust about our new crate", "expected": "REDDIT_CREATE_REDDIT_POST"} +{"intent": "search reddit for threads about the M4 macbook", "expected": "REDDIT_SEARCH_ACROSS_SUBREDDITS"} +{"intent": "what are the comments on that reddit post?", "expected": "REDDIT_RETRIEVE_POST_COMMENTS"} +{"intent": "reply to the top comment on my reddit post", "expected": "REDDIT_POST_REDDIT_COMMENT"} +{"intent": "publish a post on our facebook page about the sale", "expected": "FACEBOOK_CREATE_POST"} +{"intent": "how are our facebook page posts performing?", "expected": "FACEBOOK_GET_PAGE_INSIGHTS"} +{"intent": "which facebook pages do I manage?", "expected": "FACEBOOK_LIST_MANAGED_PAGES"} +{"intent": "show me the comments on my latest instagram post", "expected": "INSTAGRAM_GET_POST_COMMENTS"} +{"intent": "how many followers and reach did my instagram get this week?", "expected": "INSTAGRAM_GET_USER_INSIGHTS"} +{"intent": "prepare an instagram post with this photo", "expected": "INSTAGRAM_CREATE_MEDIA_CONTAINER"} +{"intent": "open src/main.rs and show me the contents", "expected": "file_read"} +{"intent": "what's in the README?", "expected": "file_read"} +{"intent": "find every place we call parse_config", "expected": "grep"} +{"intent": "which files mention TODO in the crates folder?", "expected": "grep"} +{"intent": "list all the .toml files in the repo", "expected": "glob"} +{"intent": "what's in the current directory?", "expected": "list"} +{"intent": "write a hello world script to hello.py", "expected": "file_write"} +{"intent": "apply this diff to the parser", "expected": "apply_patch"} +{"intent": "run the test suite", "expected": "shell"} +{"intent": "execute npm install", "expected": "shell"} +{"intent": "commit these changes with the message fix typo", "expected": "git_operations"} +{"intent": "what's the latest news about the fed rate decision?", "expected": "web_search_tool"} +{"intent": "google who won the champions league", "expected": "web_search_tool"} +{"intent": "fetch https://example.com/pricing and summarize it", "expected": "web_fetch"} +{"intent": "call the weather API at api.weather.example/today", "expected": "http_request"} +{"intent": "remember that I prefer short answers", "expected": "save_preference"} +{"intent": "what did I tell you about my dog?", "expected": "memory_recall"} +{"intent": "store the fact that my flight is on the 14th", "expected": "memory_store"} +{"intent": "forget what I said about the old address", "expected": "memory_forget"} +{"intent": "what time is it in Tokyo right now?", "expected": "current_time"} +{"intent": "what's next friday's date?", "expected": "resolve_time"} +{"intent": "remind me every morning at 8 to drink water", "expected": "cron_add"} +{"intent": "what scheduled jobs do I have?", "expected": "cron_list"} +{"intent": "cancel the weekly report cron", "expected": "cron_remove"} +{"intent": "add buy milk to my todo list", "expected": "todo_add"} +{"intent": "show me my todos", "expected": "todo_list"} +{"intent": "draw a picture of a cat astronaut", "expected": "create_image"} +{"intent": "make a short video of waves at sunset", "expected": "create_video"} +{"intent": "what's in this screenshot?", "expected": "analyze_image"} +{"intent": "do a deep dive on competitors in the AI note-taking space", "expected": "research"} +{"intent": "write and run a python script that sums a list", "expected": "run_code"} +{"intent": "run this python snippet: print(2**10)", "expected": "python_exec"} +{"intent": "is there a skill for summarizing PDFs?", "expected": "skill_search"} +{"intent": "search the skill registry for a stripe integration", "expected": "skill_registry_search"} +{"intent": "install the skill from https://github.com/x/y", "expected": "install_workflow_from_url"} +{"intent": "what workflows have I saved?", "expected": "list_workflows"} +{"intent": "run the weekly digest workflow", "expected": "run_workflow"} +{"intent": "check if there's an app update", "expected": "update_check"} +{"intent": "update the app to the newest version", "expected": "update_apply"} +{"intent": "set a goal to ship v2 by december", "expected": "goal_set"} +{"intent": "what's my current goal?", "expected": "goal_get"} +{"intent": "is the app healthy? run diagnostics", "expected": "doctor_health"} +{"intent": "how much have I spent on AI this month?", "expected": "cost_get_summary"} +{"intent": "find an MCP server for postgres", "expected": "mcp_registry_search"} +{"intent": "call the query tool on the postgres mcp server", "expected": "mcp_registry_tool_call"} +{"intent": "connect my hubspot account", "expected": "oauth_connect_url"} +{"intent": "export these results as a csv", "expected": "csv_export"} +{"intent": "send a pushover notification to my phone", "expected": "pushover"} +{"intent": "unsubscribe me from all these marketing emails", "expected": "gmail_unsubscribe"} +{"intent": "swap 0.1 eth for usdc", "expected": "do_crypto"} +{"intent": "schedule the report to run tomorrow at noon", "expected": "schedule_task"} +{"intent": "how do I set up the proxy in openhuman?", "expected": "ask_docs"} +{"intent": "what does the persona file say about me?", "expected": "workspace_read_persona"} +{"intent": "what did we discuss in last month's emails about the merger?", "expected": "retrieve_memory"} +{"intent": "turn this into a reusable skill", "expected": "create_skill"} +{"intent": "are the background services running?", "expected": "service_status"} +{"intent": "run three research tasks in parallel on rust, go and zig", "expected": "spawn_parallel_agents"} +{"intent": "ask me which option I want before continuing", "expected": "ask_user_clarification"} +{"intent": "build a workflow that emails me the top HN posts daily", "expected": "build_workflow"} +{"intent": "suggest workflows I could automate", "expected": "suggest_workflows"} +{"intent": "pull my open tasks from linear", "expected": "task_source_list_tasks"} +{"intent": "what facets have you learned about me?", "expected": "learning_list_facets"} +{"intent": "list the artifacts from earlier today", "expected": "artifact_list"} +{"intent": "set my status to away", "expected": "none"} +{"intent": "hi!", "expected": "none"} +{"intent": "thanks, that's all", "expected": "none"} +{"intent": "what's the capital of australia?", "expected": "none"} +{"intent": "explain the difference between tcp and udp", "expected": "none"} +{"intent": "tell me a joke", "expected": "none"} +{"intent": "how are you today?", "expected": "none"} +{"intent": "what's 17 times 23?", "expected": "none"} +{"intent": "translate 'good morning' into spanish", "expected": "none"} +{"intent": "write a haiku about autumn", "expected": "none"} +{"intent": "who wrote pride and prejudice?", "expected": "none"} +{"intent": "summarize what you just said in one sentence", "expected": "none"} +{"intent": "what's a good name for a golden retriever?", "expected": "none"} +{"intent": "can you rephrase that more formally?", "expected": "none"} +{"intent": "why is the sky blue?", "expected": "none"} +{"intent": "give me three tips for better sleep", "expected": "none"} +{"intent": "ok", "expected": "none"} +{"intent": "what does HTTP stand for?", "expected": "none"} +{"intent": "is 97 a prime number?", "expected": "none"} +{"intent": "define the word 'ephemeral'", "expected": "none"} +{"intent": "what year did the berlin wall fall?", "expected": "none"} +{"intent": "help me think through whether to take the job offer", "expected": "none"} +{"intent": "write a limerick about rust borrow checker", "expected": "none"} +{"intent": "what's the plural of octopus?", "expected": "none"} +{"intent": "convert 5 miles to kilometers", "expected": "none"} +{"intent": "recommend a sci-fi novel", "expected": "none"} +{"intent": "sorry, ignore that last message", "expected": "none"} +{"intent": "what's your name?", "expected": "none"} +{"intent": "how many days are in a leap year?", "expected": "none"} +{"intent": "explain recursion like I'm five", "expected": "none"} +{"intent": "good night", "expected": "none"} From e9629e22c950659b2c4220f20cd08e60b0aff2ac Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 22 Sep 2026 07:11:55 +0530 Subject: [PATCH 02/14] feat(tool_search_bench): add per-source recall and top-k metrics The benchmark now tracks and reports accuracy separately for composio and core tool sources, and falls back to the signed-in TinyHumans session when no API key is set, matching product behavior. Auto-committed-on: macbook --- .../src/bin/tool_search_bench.rs | 50 +++++++++++++++++-- 1 file changed, 45 insertions(+), 5 deletions(-) diff --git a/crates/openhuman-cli/src/bin/tool_search_bench.rs b/crates/openhuman-cli/src/bin/tool_search_bench.rs index 7d5f2013eb..2a541574d3 100644 --- a/crates/openhuman-cli/src/bin/tool_search_bench.rs +++ b/crates/openhuman-cli/src/bin/tool_search_bench.rs @@ -112,6 +112,10 @@ struct RankerReport { usd: f64, /// `expected family -> top-1 family -> count`, labelled rows only. confusion: BTreeMap>, + /// `source -> (labelled, top1, top3, recall@k)` where source is `composio` + /// or `core`; the connector catalogue is the heavy one, so it is read on + /// its own. + by_source: BTreeMap, misses: Vec, } @@ -287,6 +291,9 @@ fn jev_ranker(retrieval_k: usize) -> Option<(Arc, Arc Result<()> { if want("jev") { match jev_ranker(args.retrieval_k) { Some((ranker, _)) => rankers.push(("jev".into(), ranker)), - None => eprintln!( - "jev: skipped (set OPENHUMAN_BACKEND_API_KEY or TYPESAFE_API_KEY; build with the `jev` feature)" - ), + None => { + #[cfg(feature = "jev")] + { + let ranker = openhuman_tinyhumans::jev::TinyHumansJevRanker::with_config( + tinytools_jev::JevRankerConfig::new() + .with_retrieval_k(args.retrieval_k) + .with_timeout(Duration::from_secs(15)), + ); + rankers.push(("jev".into(), Arc::new(ranker))); + } + #[cfg(not(feature = "jev"))] + eprintln!("jev: skipped (build with the `jev` feature)"); + } } } @@ -399,11 +416,25 @@ async fn main() -> Result<()> { continue; } report.labelled += 1; - if got.first().map(String::as_str) == Some(row.expected.as_str()) { + let source = if catalogue + .iter() + .any(|e| e.name == row.expected && e.family.as_deref().is_some_and(|f| FIXTURE_TOOLKITS.contains(&f))) + { + "composio" + } else { + "core" + }; + let bucket = report.by_source.entry(source.to_string()).or_default(); + bucket.0 += 1; + let hit1 = got.first().map(String::as_str) == Some(row.expected.as_str()); + let hit3 = got.iter().any(|g| g == &row.expected); + if hit1 { report.top1 += 1; + bucket.1 += 1; } - if got.iter().any(|g| g == &row.expected) { + if hit3 { report.top3 += 1; + bucket.2 += 1; } else if args.misses { report.misses.push(Miss { intent: row.intent.clone(), @@ -414,6 +445,7 @@ async fn main() -> Result<()> { let retrieved = Bm25Ranker::rank_sync(&candidates, &row.intent, args.retrieval_k); if retrieved.iter().any(|h| h.key == row.expected) { report.recall_at_20 += 1; + report.by_source.get_mut(source).map(|b| b.3 += 1); } let expected_family = row .family @@ -486,6 +518,14 @@ async fn main() -> Result<()> { if r.usd == 0.0 { "-".to_string() } else { format!("${:.5}", r.usd) }, ); } + println!("\n| ranker | source | labelled | top-1 | top-3 | recall@{} |", args.retrieval_k); + println!("|---|---|---|---|---|---|"); + for r in &reports { + for (source, (n, t1, t3, rk)) in &r.by_source { + let pct = |x: usize| if *n == 0 { "n/a".to_string() } else { format!("{:.1}%", 100.0 * x as f64 / *n as f64) }; + println!("| {} | {} | {} | {} | {} | {} |", r.ranker, source, n, pct(*t1), pct(*t3), pct(*rk)); + } + } for r in &reports { println!("\n### {} — top-1 family confusion (expected → got)", r.ranker); for (expected, gots) in &r.confusion { From ee6cd6679982eab15b7fc37aa6187736f855f3ec Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 22 Sep 2026 07:24:58 +0530 Subject: [PATCH 03/14] chore(deps): switch tinyjevclient to local path for bench experiment The tinyjevclient dependency is temporarily pointed at a local path as part of a benchmark experiment testing size-aware probability tolerance, pending upstream PR. This change is not intended for production and should be reverted once the experiment concludes. Auto-committed-on: macbook --- Cargo.lock | 1 - Cargo.toml | 4 ++++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index a2edec8b83..5d8ba82e31 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6850,7 +6850,6 @@ dependencies = [ [[package]] name = "tinyjevclient" version = "0.2.1" -source = "git+https://github.com/tinyhumansai/tinyjevclient?rev=e53d5f088ff03fa38c53bac219dab7697b5016c9#e53d5f088ff03fa38c53bac219dab7697b5016c9" dependencies = [ "httpdate", "reqwest", diff --git a/Cargo.toml b/Cargo.toml index 109a4839c8..05224dbb6c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -81,3 +81,7 @@ debug = false [profile.dev.package."*"] debug = false + +# TEMP bench experiment: size-aware probability tolerance (tinyjevclient PR pending) +[patch."https://github.com/tinyhumansai/tinyjevclient"] +tinyjevclient = { path = "/private/tmp/claude-501/-Users-enamakel-work-workflow-opencompany-openhuman/e01cc3ec-1f27-4ce6-9afb-36587e79ae28/scratchpad/tinyjevclient/crates/tinyjevclient" } From 64ed46b61de6d679de162993460c6e48bf19cab9 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 22 Sep 2026 07:34:32 +0530 Subject: [PATCH 04/14] chore: files changed crates/openhuman-cli/src/bin/tool_search_bench.rs Auto-committed-on: macbook --- .../src/bin/tool_search_bench.rs | 43 +++++++++++++------ 1 file changed, 31 insertions(+), 12 deletions(-) diff --git a/crates/openhuman-cli/src/bin/tool_search_bench.rs b/crates/openhuman-cli/src/bin/tool_search_bench.rs index 2a541574d3..779b37461a 100644 --- a/crates/openhuman-cli/src/bin/tool_search_bench.rs +++ b/crates/openhuman-cli/src/bin/tool_search_bench.rs @@ -146,6 +146,7 @@ struct Args { top_k: usize, retrieval_k: usize, misses: bool, + family: bool, } fn parse_args() -> Args { @@ -157,6 +158,7 @@ fn parse_args() -> Args { top_k: 3, retrieval_k: 20, misses: false, + family: false, }; let mut it = std::env::args().skip(1); while let Some(arg) = it.next() { @@ -170,10 +172,11 @@ fn parse_args() -> Args { args.retrieval_k = it.next().and_then(|v| v.parse().ok()).unwrap_or(20) } "--misses" => args.misses = true, + "--family" => args.family = true, "-h" | "--help" => { eprintln!( "usage: tool-search-bench [--ranker all|bm25|overlap|jev] [--intents FILE] \ - [--dump-catalogue] [--json OUT] [--top-k N] [--retrieval-k N] [--misses]" + [--dump-catalogue] [--json OUT] [--top-k N] [--retrieval-k N] [--misses] [--family]" ); std::process::exit(0); } @@ -278,7 +281,10 @@ fn load_intents(path: &PathBuf) -> Result> { use openhuman_core::agent::tinyagents::discovery::OverlapRanker; #[cfg(feature = "jev")] -fn jev_ranker(retrieval_k: usize) -> Option<(Arc, Arc)> { +fn jev_ranker( + retrieval_k: usize, + family: bool, +) -> Option<(Arc, Arc)> { use tinytools_jev::{ClientConfig, JevRanker, JevRankerConfig}; let client = if let Ok(key) = std::env::var("OPENHUMAN_BACKEND_API_KEY") { let mut client = ClientConfig::tinyhumans_openrouter(key); @@ -298,17 +304,28 @@ fn jev_ranker(retrieval_k: usize) -> Option<(Arc, Arc, ranker)) } +#[cfg(feature = "jev")] +fn jev_config(retrieval_k: usize, family: bool) -> tinytools_jev::JevRankerConfig { + use tinytools_jev::{JevRankerConfig, JevStrategy}; + let config = JevRankerConfig::new() + .with_retrieval_k(retrieval_k) + .with_timeout(Duration::from_secs(20)); + if family { + config.with_strategy(JevStrategy::FamilyThenDecide) + } else { + config + } +} + #[cfg(not(feature = "jev"))] -fn jev_ranker(_retrieval_k: usize) -> Option<(Arc, Arc<()>)> { +fn jev_ranker(_retrieval_k: usize, _family: bool) -> Option<(Arc, Arc<()>)> { None } @@ -366,15 +383,13 @@ async fn main() -> Result<()> { rankers.push(("overlap".into(), Arc::new(OverlapRanker))); } if want("jev") { - match jev_ranker(args.retrieval_k) { + match jev_ranker(args.retrieval_k, args.family) { Some((ranker, _)) => rankers.push(("jev".into(), ranker)), None => { #[cfg(feature = "jev")] { let ranker = openhuman_tinyhumans::jev::TinyHumansJevRanker::with_config( - tinytools_jev::JevRankerConfig::new() - .with_retrieval_k(args.retrieval_k) - .with_timeout(Duration::from_secs(15)), + jev_config(args.retrieval_k, args.family), ); rankers.push(("jev".into(), Arc::new(ranker))); } @@ -387,7 +402,11 @@ async fn main() -> Result<()> { let mut reports = Vec::new(); for (kind, ranker) in &rankers { let mut report = RankerReport { - ranker: kind.clone(), + ranker: if kind == "jev" && args.family { + "jev(family)".to_string() + } else { + kind.clone() + }, rows: rows.len(), ..RankerReport::default() }; @@ -469,7 +488,7 @@ async fn main() -> Result<()> { if let Some(report) = reports.iter_mut().find(|r| r.ranker == "jev") { // Tokens and cost: one detailed pass over the labelled rows so the // number is the provider's own `usage`, not an estimate. - if let Some((_, detailed)) = jev_ranker(args.retrieval_k) { + if let Some((_, detailed)) = jev_ranker(args.retrieval_k, args.family) { let mut tokens = 0_u64; let mut counted = 0_u64; for row in rows.iter().take(25) { From bdb31c64da394bed861b11cf56620cce170b88f6 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 22 Sep 2026 07:42:53 +0530 Subject: [PATCH 05/14] refactor(core): move discovery module into subdirectory The discovery module is reorganized into a subdirectory to accommodate the new embedding ranker component, which is added as a separate module and re-exported for public use. Auto-committed-on: macbook --- .../{ => discovery}/discovery_tests.rs | 0 .../tinyagents/discovery/embedding_ranker.rs | 257 ++++++++++++++++++ .../discovery/embedding_ranker_tests.rs | 120 ++++++++ .../{discovery.rs => discovery/mod.rs} | 4 + 4 files changed, 381 insertions(+) rename crates/openhuman-core/src/agent/tinyagents/{ => discovery}/discovery_tests.rs (100%) create mode 100644 crates/openhuman-core/src/agent/tinyagents/discovery/embedding_ranker.rs create mode 100644 crates/openhuman-core/src/agent/tinyagents/discovery/embedding_ranker_tests.rs rename crates/openhuman-core/src/agent/tinyagents/{discovery.rs => discovery/mod.rs} (98%) diff --git a/crates/openhuman-core/src/agent/tinyagents/discovery_tests.rs b/crates/openhuman-core/src/agent/tinyagents/discovery/discovery_tests.rs similarity index 100% rename from crates/openhuman-core/src/agent/tinyagents/discovery_tests.rs rename to crates/openhuman-core/src/agent/tinyagents/discovery/discovery_tests.rs diff --git a/crates/openhuman-core/src/agent/tinyagents/discovery/embedding_ranker.rs b/crates/openhuman-core/src/agent/tinyagents/discovery/embedding_ranker.rs new file mode 100644 index 0000000000..20c4c817b0 --- /dev/null +++ b/crates/openhuman-core/src/agent/tinyagents/discovery/embedding_ranker.rs @@ -0,0 +1,257 @@ +//! [`EmbeddingToolRanker`]: the semantic retriever for `tool_search`. +//! +//! A lexical retriever loses every paraphrase — "ping alex" never overlaps +//! `SLACK_SEND_MESSAGE send a message` — and a decision model can only pick +//! from what the retriever hands it. Measured on the orchestrator's catalogue +//! plus nine Composio toolkits (1,215 tools), BM25 recall@20 was 70%, and +//! that ceiling capped Jev at 67% top-3. This ranker embeds each tool's +//! summary once with the process's configured embedding provider (the same +//! one memory recall uses), embeds the intent per search, and ranks by cosine +//! similarity, so recall follows meaning rather than shared words. It is the +//! `retriever` inside `tinytools_jev::JevRanker` for every catalogue larger +//! than one Jev `Choice`. +//! +//! Catalogue embeddings are cached in memory by content hash and, when a +//! cache path is given, on disk keyed by the provider's signature, so a cold +//! process pays for the catalogue once and every later search embeds only +//! the intent (one provider call). + +use std::collections::HashMap; +use std::hash::{Hash, Hasher}; +use std::path::PathBuf; +use std::sync::{Arc, RwLock}; + +use serde::{Deserialize, Serialize}; +use tinytools::{RankCandidate, RankContext, RankError, RankHit, ToolRanker}; + +use crate::inference::embedding_host::EmbeddingProvider; + +/// Texts per embedding request. The managed provider accepts far more, but a +/// bounded batch keeps one request under any body cap and lets a partial +/// failure cost one batch, not the catalogue. +const EMBED_BATCH: usize = 64; + +/// Ranks tools by cosine similarity between the intent and each tool's +/// summary, with catalogue embeddings cached. +pub struct EmbeddingToolRanker { + provider: Arc, + cache: RwLock>>, + cache_path: Option, +} + +#[derive(Serialize, Deserialize, Default)] +struct DiskCache { + signature: String, + entries: HashMap>, +} + +impl std::fmt::Debug for EmbeddingToolRanker { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("EmbeddingToolRanker") + .field("provider", &self.provider.name()) + .field("model", &self.provider.model_id()) + .field("cache_path", &self.cache_path) + .finish_non_exhaustive() + } +} + +impl EmbeddingToolRanker { + /// The stable [`ToolRanker::kind`] of this ranker. + pub const KIND: &'static str = "embedding"; + + /// A ranker over `provider` with an in-memory cache only. + pub fn new(provider: Arc) -> Self { + Self { + provider, + cache: RwLock::new(HashMap::new()), + cache_path: None, + } + } + + /// A ranker whose catalogue embeddings also persist at `path`, keyed by + /// the provider's signature so a model change invalidates them. A + /// missing or unreadable file is an empty cache, never an error. + pub fn with_disk_cache(mut self, path: PathBuf) -> Self { + if let Ok(raw) = std::fs::read(&path) + && let Ok(disk) = serde_json::from_slice::(&raw) + && disk.signature == self.provider.signature() + { + tracing::debug!( + entries = disk.entries.len(), + path = %path.display(), + "[tool-search] loaded embedding cache" + ); + *self.cache.write().unwrap_or_else(|p| p.into_inner()) = disk.entries; + } + self.cache_path = Some(path); + self + } + + /// Whether `provider` can embed at all. The `none` provider embeds + /// nothing and would rank everything at zero. + pub fn provider_is_usable(provider: &dyn EmbeddingProvider) -> bool { + provider.dimensions() > 0 && provider.name() != "none" + } + + fn key(candidate: &RankCandidate) -> u64 { + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + candidate.summary.hash(&mut hasher); + candidate.family.hash(&mut hasher); + hasher.finish() + } + + fn text(candidate: &RankCandidate) -> String { + match &candidate.family { + Some(family) => format!("{} ({family})", candidate.summary), + None => candidate.summary.clone(), + } + } + + /// Embeds every candidate not already cached, in batches. + async fn ensure_cached(&self, candidates: &[RankCandidate]) -> Result<(), RankError> { + let missing: Vec<(u64, String)> = { + let cache = self.cache.read().unwrap_or_else(|p| p.into_inner()); + let mut seen = std::collections::HashSet::new(); + candidates + .iter() + .map(|c| (Self::key(c), c)) + .filter(|(k, _)| !cache.contains_key(k) && seen.insert(*k)) + .map(|(k, c)| (k, Self::text(c))) + .collect() + }; + if missing.is_empty() { + return Ok(()); + } + tracing::info!( + missing = missing.len(), + provider = self.provider.name(), + "[tool-search] embedding catalogue entries" + ); + for batch in missing.chunks(EMBED_BATCH) { + let texts: Vec<&str> = batch.iter().map(|(_, t)| t.as_str()).collect(); + let vectors = self + .provider + .embed(&texts) + .await + .map_err(|error| RankError::Backend { + reason: format!("embedding failed: {error:#}"), + })?; + if vectors.len() != batch.len() { + return Err(RankError::Backend { + reason: format!( + "embedding returned {} vectors for {} texts", + vectors.len(), + batch.len() + ), + }); + } + let mut cache = self.cache.write().unwrap_or_else(|p| p.into_inner()); + for ((key, _), vector) in batch.iter().zip(vectors) { + cache.insert(*key, vector); + } + } + self.persist(); + Ok(()) + } + + fn persist(&self) { + let Some(path) = &self.cache_path else { + return; + }; + let disk = DiskCache { + signature: self.provider.signature(), + entries: self + .cache + .read() + .unwrap_or_else(|p| p.into_inner()) + .clone(), + }; + let write = || -> std::io::Result<()> { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent)?; + } + let tmp = path.with_extension("json.tmp"); + std::fs::write(&tmp, serde_json::to_vec(&disk)?)?; + std::fs::rename(&tmp, path) + }; + if let Err(error) = write() { + tracing::warn!( + path = %path.display(), + error = %error, + "[tool-search] could not persist embedding cache" + ); + } + } +} + +fn cosine(a: &[f32], b: &[f32]) -> f64 { + let (mut dot, mut na, mut nb) = (0.0_f64, 0.0_f64, 0.0_f64); + for (x, y) in a.iter().zip(b) { + dot += f64::from(*x) * f64::from(*y); + na += f64::from(*x) * f64::from(*x); + nb += f64::from(*y) * f64::from(*y); + } + if na == 0.0 || nb == 0.0 { + return 0.0; + } + dot / (na.sqrt() * nb.sqrt()) +} + +#[async_trait::async_trait] +impl ToolRanker for EmbeddingToolRanker { + fn kind(&self) -> &'static str { + Self::KIND + } + + async fn rank( + &self, + intent: &str, + _context: &RankContext, + candidates: &[RankCandidate], + limit: usize, + ) -> Result, RankError> { + let intent = intent.trim(); + if intent.is_empty() { + return Err(RankError::InvalidInput { + reason: "intent is empty".to_owned(), + }); + } + if candidates.is_empty() || limit == 0 { + return Ok(Vec::new()); + } + self.ensure_cached(candidates).await?; + let query = self + .provider + .embed(&[intent]) + .await + .map_err(|error| RankError::Backend { + reason: format!("embedding failed: {error:#}"), + })? + .into_iter() + .next() + .ok_or_else(|| RankError::Backend { + reason: "embedding returned no vector for the intent".to_owned(), + })?; + let cache = self.cache.read().unwrap_or_else(|p| p.into_inner()); + let mut scored: Vec = candidates + .iter() + .filter_map(|c| { + cache + .get(&Self::key(c)) + .map(|v| RankHit::new(c.key.clone(), cosine(&query, v))) + }) + .collect(); + scored.sort_by(|a, b| { + b.score + .partial_cmp(&a.score) + .unwrap_or(std::cmp::Ordering::Equal) + .then_with(|| a.key.cmp(&b.key)) + }); + scored.truncate(limit); + Ok(scored) + } +} + +#[cfg(test)] +#[path = "embedding_ranker_tests.rs"] +mod tests; diff --git a/crates/openhuman-core/src/agent/tinyagents/discovery/embedding_ranker_tests.rs b/crates/openhuman-core/src/agent/tinyagents/discovery/embedding_ranker_tests.rs new file mode 100644 index 0000000000..5243f927b6 --- /dev/null +++ b/crates/openhuman-core/src/agent/tinyagents/discovery/embedding_ranker_tests.rs @@ -0,0 +1,120 @@ +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; + +use tinytools::{RankCandidate, RankContext, ToolRanker}; + +use super::*; + +/// Embeds a text as a bag of three hand-picked words, so similarity is +/// deterministic and readable. +struct BagEmbedder { + calls: AtomicUsize, +} + +#[async_trait::async_trait] +impl EmbeddingProvider for BagEmbedder { + fn name(&self) -> &str { + "bag" + } + fn model_id(&self) -> &str { + "bag-v1" + } + fn dimensions(&self) -> usize { + 3 + } + fn signature(&self) -> String { + "provider=bag;model=bag-v1;dims=3".into() + } + async fn embed(&self, texts: &[&str]) -> anyhow::Result>> { + self.calls.fetch_add(1, Ordering::SeqCst); + Ok(texts + .iter() + .map(|t| { + let t = t.to_ascii_lowercase(); + vec![ + f32::from(u8::from(t.contains("message") || t.contains("ping"))), + f32::from(u8::from(t.contains("email") || t.contains("mail"))), + f32::from(u8::from(t.contains("file"))), + ] + }) + .collect()) + } +} + +fn candidates() -> Vec { + vec![ + RankCandidate::new("SLACK_SEND_MESSAGE", "send a message to a channel").with_family("slack"), + RankCandidate::new("GMAIL_SEND_EMAIL", "send an email").with_family("gmail"), + RankCandidate::new("file_read", "read a file"), + ] +} + +#[tokio::test] +async fn ranks_by_cosine_and_embeds_the_catalogue_once() { + let embedder = Arc::new(BagEmbedder { + calls: AtomicUsize::new(0), + }); + let ranker = EmbeddingToolRanker::new(embedder.clone()); + assert_eq!(ranker.kind(), "embedding"); + + let hits = ranker + .rank("ping alex", &RankContext::empty(), &candidates(), 2) + .await + .unwrap(); + assert_eq!(hits[0].key, "SLACK_SEND_MESSAGE"); + assert!(hits[0].confidence.is_none()); + assert_eq!(hits.len(), 2); + // One batch for the catalogue plus one for the intent. + assert_eq!(embedder.calls.load(Ordering::SeqCst), 2); + + let hits = ranker + .rank("mail the report", &RankContext::empty(), &candidates(), 1) + .await + .unwrap(); + assert_eq!(hits[0].key, "GMAIL_SEND_EMAIL"); + // Only the intent was embedded this time. + assert_eq!(embedder.calls.load(Ordering::SeqCst), 3); +} + +#[tokio::test] +async fn disk_cache_round_trips_and_is_keyed_by_signature() { + let tmp = tempfile::tempdir().unwrap(); + let path = tmp.path().join("cache").join("tool_search_embeddings.json"); + let embedder = Arc::new(BagEmbedder { + calls: AtomicUsize::new(0), + }); + let ranker = EmbeddingToolRanker::new(embedder.clone()).with_disk_cache(path.clone()); + ranker + .rank("ping", &RankContext::empty(), &candidates(), 1) + .await + .unwrap(); + assert!(path.exists()); + + let embedder2 = Arc::new(BagEmbedder { + calls: AtomicUsize::new(0), + }); + let warm = EmbeddingToolRanker::new(embedder2.clone()).with_disk_cache(path.clone()); + warm.rank("ping", &RankContext::empty(), &candidates(), 1) + .await + .unwrap(); + assert_eq!( + embedder2.calls.load(Ordering::SeqCst), + 1, + "a warm cache embeds only the intent" + ); +} + +#[tokio::test] +async fn empty_intent_is_rejected_and_none_provider_is_unusable() { + let ranker = EmbeddingToolRanker::new(Arc::new(BagEmbedder { + calls: AtomicUsize::new(0), + })); + assert!(ranker + .rank(" ", &RankContext::empty(), &candidates(), 1) + .await + .is_err()); + let none = crate::inference::embedding_host::TinyInferenceEmbeddingProvider::new( + tinyinference_embeddings::NoopEmbeddingModel, + ); + assert!(!EmbeddingToolRanker::provider_is_usable(&none)); +} diff --git a/crates/openhuman-core/src/agent/tinyagents/discovery.rs b/crates/openhuman-core/src/agent/tinyagents/discovery/mod.rs similarity index 98% rename from crates/openhuman-core/src/agent/tinyagents/discovery.rs rename to crates/openhuman-core/src/agent/tinyagents/discovery/mod.rs index 92807ddbb7..f77dafa80d 100644 --- a/crates/openhuman-core/src/agent/tinyagents/discovery.rs +++ b/crates/openhuman-core/src/agent/tinyagents/discovery/mod.rs @@ -168,6 +168,10 @@ impl ToolRanker for OverlapRanker { } } +mod embedding_ranker; + +pub use embedding_ranker::EmbeddingToolRanker; + #[cfg(test)] #[path = "discovery_tests.rs"] mod tests; From 3eca97deb815ed28b0855169e4eafc08f366f7d7 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 22 Sep 2026 07:47:14 +0530 Subject: [PATCH 06/14] chore: refactor disk cache loading to use combinators Refactored the disk cache loading logic in `with_disk_cache` to use a chain of combinators (`ok`, `and_then`, `filter`) instead of nested `if let` conditions. This makes the control flow more linear and readable while preserving the same behavior: only a valid, signature-matching cache file is loaded, and any failure results in an empty cache. Auto-committed-on: macbook --- .../src/agent/tinyagents/discovery/embedding_ranker.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/crates/openhuman-core/src/agent/tinyagents/discovery/embedding_ranker.rs b/crates/openhuman-core/src/agent/tinyagents/discovery/embedding_ranker.rs index 20c4c817b0..5ba8c1f4cf 100644 --- a/crates/openhuman-core/src/agent/tinyagents/discovery/embedding_ranker.rs +++ b/crates/openhuman-core/src/agent/tinyagents/discovery/embedding_ranker.rs @@ -72,10 +72,11 @@ impl EmbeddingToolRanker { /// the provider's signature so a model change invalidates them. A /// missing or unreadable file is an empty cache, never an error. pub fn with_disk_cache(mut self, path: PathBuf) -> Self { - if let Ok(raw) = std::fs::read(&path) - && let Ok(disk) = serde_json::from_slice::(&raw) - && disk.signature == self.provider.signature() - { + let loaded = std::fs::read(&path) + .ok() + .and_then(|raw| serde_json::from_slice::(&raw).ok()) + .filter(|disk| disk.signature == self.provider.signature()); + if let Some(disk) = loaded { tracing::debug!( entries = disk.entries.len(), path = %path.display(), From 0f39191f18ea989224ba9c3850901fcf968e5511 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 22 Sep 2026 07:49:45 +0530 Subject: [PATCH 07/14] feat(jev): add embedding retriever option to tool search bench The tool search benchmark now supports an `--embedding` flag that switches the Jev ranker's retriever from BM25 to the process's configured embedding provider, with a disk cache so the catalogue is embedded only once per process. The TinyHumansJevRanker also reuses the embedding retriever across rebuilds, falling back to BM25 when the provider cannot embed, so family decisions are cut by meaning rather than shared words. Auto-committed-on: macbook --- .../src/bin/tool_search_bench.rs | 71 +++++++++++++++---- crates/openhuman-tinyhumans/src/jev/ranker.rs | 46 +++++++++++- 2 files changed, 102 insertions(+), 15 deletions(-) diff --git a/crates/openhuman-cli/src/bin/tool_search_bench.rs b/crates/openhuman-cli/src/bin/tool_search_bench.rs index 779b37461a..18fdcf8ff1 100644 --- a/crates/openhuman-cli/src/bin/tool_search_bench.rs +++ b/crates/openhuman-cli/src/bin/tool_search_bench.rs @@ -147,6 +147,7 @@ struct Args { retrieval_k: usize, misses: bool, family: bool, + embedding: bool, } fn parse_args() -> Args { @@ -159,6 +160,7 @@ fn parse_args() -> Args { retrieval_k: 20, misses: false, family: false, + embedding: false, }; let mut it = std::env::args().skip(1); while let Some(arg) = it.next() { @@ -173,10 +175,11 @@ fn parse_args() -> Args { } "--misses" => args.misses = true, "--family" => args.family = true, + "--embedding" => args.embedding = true, "-h" | "--help" => { eprintln!( - "usage: tool-search-bench [--ranker all|bm25|overlap|jev] [--intents FILE] \ - [--dump-catalogue] [--json OUT] [--top-k N] [--retrieval-k N] [--misses] [--family]" + "usage: tool-search-bench [--ranker all|bm25|overlap|embedding|jev] [--intents FILE] \ + [--dump-catalogue] [--json OUT] [--top-k N] [--retrieval-k N] [--misses] [--family] [--embedding]" ); std::process::exit(0); } @@ -284,6 +287,7 @@ use openhuman_core::agent::tinyagents::discovery::OverlapRanker; fn jev_ranker( retrieval_k: usize, family: bool, + embedding: bool, ) -> Option<(Arc, Arc)> { use tinytools_jev::{ClientConfig, JevRanker, JevRankerConfig}; let client = if let Ok(key) = std::env::var("OPENHUMAN_BACKEND_API_KEY") { @@ -304,7 +308,7 @@ fn jev_ranker( }; let ranker = JevRanker::from_config( client, - jev_config(retrieval_k, family), + jev_config(retrieval_k, family, embedding), ) .ok()?; let ranker = Arc::new(ranker); @@ -312,23 +316,55 @@ fn jev_ranker( } #[cfg(feature = "jev")] -fn jev_config(retrieval_k: usize, family: bool) -> tinytools_jev::JevRankerConfig { +fn jev_config(retrieval_k: usize, family: bool, embedding: bool) -> tinytools_jev::JevRankerConfig { use tinytools_jev::{JevRankerConfig, JevStrategy}; - let config = JevRankerConfig::new() + let mut config = JevRankerConfig::new() .with_retrieval_k(retrieval_k) .with_timeout(Duration::from_secs(20)); if family { - config.with_strategy(JevStrategy::FamilyThenDecide) - } else { - config + config = config.with_strategy(JevStrategy::FamilyThenDecide); + } + if embedding { + config = config.with_retriever(embedding_retriever()); } + config } #[cfg(not(feature = "jev"))] -fn jev_ranker(_retrieval_k: usize, _family: bool) -> Option<(Arc, Arc<()>)> { +fn jev_ranker( + _retrieval_k: usize, + _family: bool, + _embedding: bool, +) -> Option<(Arc, Arc<()>)> { None } +/// The process's configured embedding provider as a `ToolRanker`, with its +/// catalogue cache in the scratch workspace so repeated runs embed only the +/// intents. +fn embedding_retriever() -> Arc { + use openhuman_core::agent::tinyagents::discovery::EmbeddingToolRanker; + let config = tokio::task::block_in_place(|| { + tokio::runtime::Handle::current().block_on(openhuman_core::config::Config::load_or_init()) + }) + .expect("load config for the embedding provider"); + let provider = + openhuman_core::inference::embedding_host::default_embedding_provider_with_config(&config); + if !EmbeddingToolRanker::provider_is_usable(provider.as_ref()) { + eprintln!( + "embedding: provider `{}` cannot embed; the bench retriever stays bm25", + provider.name() + ); + return Arc::new(Bm25Ranker); + } + eprintln!("embedding: {} / {}", provider.name(), provider.model_id()); + Arc::new( + EmbeddingToolRanker::new(provider).with_disk_cache( + repo_root().join("target").join("tool_search_bench_embeddings.json"), + ), + ) +} + fn family_of<'a>(catalogue: &'a [CatalogueEntry], name: &str) -> &'a str { catalogue .iter() @@ -382,14 +418,17 @@ async fn main() -> Result<()> { if want("overlap") { rankers.push(("overlap".into(), Arc::new(OverlapRanker))); } + if want("embedding") && args.embedding { + rankers.push(("embedding".into(), embedding_retriever())); + } if want("jev") { - match jev_ranker(args.retrieval_k, args.family) { + match jev_ranker(args.retrieval_k, args.family, args.embedding) { Some((ranker, _)) => rankers.push(("jev".into(), ranker)), None => { #[cfg(feature = "jev")] { let ranker = openhuman_tinyhumans::jev::TinyHumansJevRanker::with_config( - jev_config(args.retrieval_k, args.family), + jev_config(args.retrieval_k, args.family, args.embedding), ); rankers.push(("jev".into(), Arc::new(ranker))); } @@ -402,8 +441,12 @@ async fn main() -> Result<()> { let mut reports = Vec::new(); for (kind, ranker) in &rankers { let mut report = RankerReport { - ranker: if kind == "jev" && args.family { - "jev(family)".to_string() + ranker: if kind == "jev" { + format!( + "jev({}{})", + if args.family { "family" } else { "retrieve" }, + if args.embedding { "+embedding" } else { "+bm25" } + ) } else { kind.clone() }, @@ -488,7 +531,7 @@ async fn main() -> Result<()> { if let Some(report) = reports.iter_mut().find(|r| r.ranker == "jev") { // Tokens and cost: one detailed pass over the labelled rows so the // number is the provider's own `usage`, not an estimate. - if let Some((_, detailed)) = jev_ranker(args.retrieval_k, args.family) { + if let Some((_, detailed)) = jev_ranker(args.retrieval_k, args.family, args.embedding) { let mut tokens = 0_u64; let mut counted = 0_u64; for row in rows.iter().take(25) { diff --git a/crates/openhuman-tinyhumans/src/jev/ranker.rs b/crates/openhuman-tinyhumans/src/jev/ranker.rs index 37a33d8bb5..a3ae303661 100644 --- a/crates/openhuman-tinyhumans/src/jev/ranker.rs +++ b/crates/openhuman-tinyhumans/src/jev/ranker.rs @@ -9,6 +9,7 @@ use std::{ use std::{future::Future, pin::Pin, sync::Arc}; +use openhuman_core::agent::tinyagents::discovery::EmbeddingToolRanker; use openhuman_core::api::config::effective_backend_api_url; use openhuman_core::config::Config; use openhuman_core::security::credentials::session_support::resolve_backend_credential; @@ -33,6 +34,9 @@ pub struct TinyHumansJevRanker { struct Cached { fingerprint: u64, ranker: JevRanker, + /// The retriever inside `ranker`, kept so its catalogue embeddings + /// survive a credential change. + retriever: Arc, } impl std::fmt::Debug for TinyHumansJevRanker { @@ -104,7 +108,18 @@ impl TinyHumansJevRanker { } let mut client = ClientConfig::tinyhumans_openrouter(credential.into_secret()); client.base_url = base_url.clone(); - let ranker = JevRanker::from_config(client, self.config.clone())?; + // The retriever is the process's embedding provider when it can + // embed (the same one memory recall uses), so a family larger than + // one Jev Choice is cut by meaning, not by shared words. Reused + // across rebuilds so the catalogue is embedded once per process. + let retriever: Arc = match cached.as_ref() { + Some(entry) => entry.retriever.clone(), + None => retriever_for(&config), + }; + let ranker = JevRanker::from_config( + client, + self.config.clone().with_retriever(retriever.clone()), + )?; log::info!( "[tool-search] jev ranker bound to backend {} ({})", openhuman_core::util::redact::redact_url_for_log(&base_url), @@ -117,11 +132,40 @@ impl TinyHumansJevRanker { *cached = Some(Cached { fingerprint, ranker: ranker.clone(), + retriever, }); Ok(ranker) } } +/// The semantic retriever for `config`'s embedding provider, or BM25 when +/// the provider cannot embed (`none`, or a managed provider with no route). +fn retriever_for(config: &Config) -> Arc { + let provider = openhuman_core::inference::embedding_host::default_embedding_provider_with_config( + config, + ); + if !EmbeddingToolRanker::provider_is_usable(provider.as_ref()) { + log::info!( + "[tool-search] embedding provider `{}` cannot embed; retrieving with bm25", + provider.name() + ); + return Arc::new(tinytools::Bm25Ranker); + } + log::info!( + "[tool-search] retrieving with embeddings ({} / {})", + provider.name(), + provider.model_id() + ); + Arc::new( + EmbeddingToolRanker::new(provider).with_disk_cache( + config + .workspace_dir + .join("cache") + .join("tool_search_embeddings.json"), + ), + ) +} + fn fingerprint(secret: &str, base_url: &str) -> u64 { let mut hasher = DefaultHasher::new(); secret.hash(&mut hasher); From d4a934587d4525ad02b169e01725b50c1093c125 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 22 Sep 2026 07:53:18 +0530 Subject: [PATCH 08/14] chore: remove unused JevRankerConfig import Remove the JevRankerConfig from the use statement in the JEV ranker setup, as it is no longer referenced in the code. Auto-committed-on: macbook --- crates/openhuman-cli/src/bin/tool_search_bench.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/openhuman-cli/src/bin/tool_search_bench.rs b/crates/openhuman-cli/src/bin/tool_search_bench.rs index 18fdcf8ff1..8862f30ca8 100644 --- a/crates/openhuman-cli/src/bin/tool_search_bench.rs +++ b/crates/openhuman-cli/src/bin/tool_search_bench.rs @@ -289,7 +289,7 @@ fn jev_ranker( family: bool, embedding: bool, ) -> Option<(Arc, Arc)> { - use tinytools_jev::{ClientConfig, JevRanker, JevRankerConfig}; + use tinytools_jev::{ClientConfig, JevRanker}; let client = if let Ok(key) = std::env::var("OPENHUMAN_BACKEND_API_KEY") { let mut client = ClientConfig::tinyhumans_openrouter(key); if let Ok(base) = std::env::var("BACKEND_URL") { From 6028b7a14b87187d39d3569963fecdf59c419152 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 22 Sep 2026 08:58:41 +0530 Subject: [PATCH 09/14] feat(jev): add configurable deadline for JEV evaluations The TinyHumansJevRanker now supports a per-evaluation deadline, defaulting to six seconds, which is applied to the TinyJevEvaluator. This bounds slow proxy responses and triggers a BM25 fallback within the turn, while the benchmark tool sets a 20-second deadline for its runs. The change also simplifies the recall accounting in the benchmark by using an if-let binding instead of map. Auto-committed-on: macbook --- .../src/bin/tool_search_bench.rs | 7 +++++-- crates/openhuman-tinyhumans/src/jev/ranker.rs | 20 +++++++++++++++++-- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/crates/openhuman-cli/src/bin/tool_search_bench.rs b/crates/openhuman-cli/src/bin/tool_search_bench.rs index ade9375c9e..5c38894d86 100644 --- a/crates/openhuman-cli/src/bin/tool_search_bench.rs +++ b/crates/openhuman-cli/src/bin/tool_search_bench.rs @@ -429,7 +429,8 @@ async fn main() -> Result<()> { { let ranker = openhuman_tinyhumans::jev::TinyHumansJevRanker::with_config( jev_config(args.retrieval_k, args.family, args.embedding), - ); + ) + .with_deadline(Duration::from_secs(20)); rankers.push(("jev".into(), Arc::new(ranker))); } #[cfg(not(feature = "jev"))] @@ -526,7 +527,9 @@ async fn main() -> Result<()> { }; if retrieved.iter().any(|h| h == &row.expected) { report.recall_at_20 += 1; - report.by_source.get_mut(source).map(|b| b.3 += 1); + if let Some(bucket) = report.by_source.get_mut(source) { + bucket.3 += 1; + } } let expected_family = row .family diff --git a/crates/openhuman-tinyhumans/src/jev/ranker.rs b/crates/openhuman-tinyhumans/src/jev/ranker.rs index fa76c9e844..33c6236261 100644 --- a/crates/openhuman-tinyhumans/src/jev/ranker.rs +++ b/crates/openhuman-tinyhumans/src/jev/ranker.rs @@ -7,7 +7,7 @@ use std::{ sync::Mutex, }; -use std::{future::Future, pin::Pin, sync::Arc}; +use std::{future::Future, pin::Pin, sync::Arc, time::Duration}; use openhuman_core::agent::tinyagents::discovery::EmbeddingToolRanker; use openhuman_core::api::config::effective_backend_api_url; @@ -31,9 +31,18 @@ pub type ConfigLoader = Arc< pub struct TinyHumansJevRanker { config: JevRankerConfig, load_config: ConfigLoader, + /// Deadline for one evaluation. Measured through the TinyHumans proxy + /// (2026-09) one evaluation takes 0.7–1.9 s at p50 and the family + /// strategy runs its second-stage evaluations concurrently, so six + /// seconds bounds a slow search well above the norm while still turning + /// a stalled proxy into a BM25 fallback inside the turn. + deadline: Duration, cached: Mutex>, } +/// Default per-evaluation deadline; see `TinyHumansJevRanker::deadline`. +const DEFAULT_DEADLINE: Duration = Duration::from_secs(6); + struct Cached { fingerprint: u64, ranker: JevRanker, @@ -72,10 +81,17 @@ impl TinyHumansJevRanker { load_config: Arc::new(|| { Box::pin(openhuman_core::config::ops::load_config_with_timeout()) }), + deadline: DEFAULT_DEADLINE, cached: Mutex::new(None), } } + /// Sets the per-evaluation deadline. + pub fn with_deadline(mut self, deadline: Duration) -> Self { + self.deadline = deadline; + self + } + /// Reads the config through `loader` instead of the core's read path. pub fn with_config_loader(mut self, loader: ConfigLoader) -> Self { self.load_config = loader; @@ -116,7 +132,7 @@ impl TinyHumansJevRanker { let client = Client::new(client_config) .map_err(|error| RankError::invalid_input(error.to_string()))?; let evaluator: Arc = - Arc::new(TinyJevEvaluator::new(client)); + Arc::new(TinyJevEvaluator::new(client).with_deadline(self.deadline)); // The retriever is the process's embedding provider when it can // embed (the same one memory recall uses), so a family larger than // one Jev Choice is cut by meaning, not by shared words. Reused From 016c1c4db836a15fac51cb147c290e562ff737d2 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 22 Sep 2026 09:00:49 +0530 Subject: [PATCH 10/14] docs(bin): document the new tool-search-bench binary Adds a row to the binary index table for `tool-search-bench`, which compares `tool_search` rankers (bm25, overlap, embedding, jev) against the real orchestrator registry and recorded Composio catalogues using the intents fixture. Auto-committed-on: macbook --- crates/openhuman-cli/src/bin/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/openhuman-cli/src/bin/README.md b/crates/openhuman-cli/src/bin/README.md index 10a9daaa01..a9fa708412 100644 --- a/crates/openhuman-cli/src/bin/README.md +++ b/crates/openhuman-cli/src/bin/README.md @@ -23,6 +23,7 @@ the arguments to `openhuman_core::run_core_from_args`. | `openhuman-fleet` | `fleet.rs` | `http-server`, `bin-tools` | Process-per-user supervisor + reverse proxy | | `rss-bench` | `rss_bench.rs` | `rss-bench` | Steady-state RSS benchmark for an embedded agent roster | | `tool-dialect-bench` | `tool_dialect_bench.rs` | none | Manual A/B of the text tool-call dialects against a local Ollama model | +| `tool-search-bench` | `tool_search_bench.rs` | none (`jev`, in `default`, for the Jev rankers) | `tool_search` ranker comparison (bm25 / overlap / embedding / jev) over the real orchestrator registry plus the recorded Composio catalogues, against `tests/fixtures/tool_search/intents.jsonl` | | `library-profile` | `library_profile/main.rs` (+ `harness.rs`, `mock.rs`, `scenarios/`) | `rss-bench` (add `rss-bench-dhat` for heap profiles) | Hermetic library-embedding profiling scenarios | `http-server` is in `default`; `bin-tools`, `rss-bench` and `rss-bench-dhat` From 2950faa1047f29edab987cc6d71d0b1bbed0d10e Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 22 Sep 2026 09:08:24 +0530 Subject: [PATCH 11/14] docs(jev-tool-search-baseline): add baseline and results for tool_search ranking Adds a planning document recording the measured baseline for the `tool_search` ranking work, including benchmark methodology, per-ranker results across 160 intents and 1,000 Composio actions, and the rationale for the product default of family-first selection with an embedding-based cut. Auto-committed-on: macbook --- docs/plans/jev-tool-search-baseline.md | 90 ++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 docs/plans/jev-tool-search-baseline.md diff --git a/docs/plans/jev-tool-search-baseline.md b/docs/plans/jev-tool-search-baseline.md new file mode 100644 index 0000000000..0b556c01e3 --- /dev/null +++ b/docs/plans/jev-tool-search-baseline.md @@ -0,0 +1,90 @@ +# `tool_search` ranking: baseline and results + +Measured 2026-09-22 with `cargo run -p openhuman-cli --bin tool-search-bench`, +live Jev (`jev-1.13` through the TinyHumans System One proxy, signed-in +session) and the managed `embedding-v1` embedder. Catalogue: every tool the +orchestrator session registers (215) plus the recorded Composio catalogues +under `tests/fixtures/composio_*.json` (1,000 actions across gmail, slack, +github, notion, googledrive, googlesheets, reddit, facebook, instagram) as the +deferred per-action tools a signed-in workspace synthesises. Intents: +`tests/fixtures/tool_search/intents.jsonl` — 160 hand-written requests, 66 +labelled with a Composio action, 63 with a core tool, 31 that no tool should +answer. + +Before this work the orchestrator reached a Composio action only through +`delegate_to_integrations_agent` → an `integrations_agent` sub-run whose +toolkit was narrowed by `rank_tools_by_prompt` (the `overlap` row). Every +`tool_search` row is one search followed by a direct call of the tool it +returns; no sub-agent. + +## Rankers + +| ranker | rows | top-1 | top-3 | retriever recall@20 | needless (of 31) | errors | p50 ms | p95 ms | +|---|---|---|---|---|---|---|---|---| +| bm25 | 160 | 22.5% | 38.0% | 70.5% | 26 | 0 | 28 | 29 | +| overlap (`rank_tools_by_prompt`, the sub-agent's narrowing today) | 160 | 35.7% | 52.7% | 69.0% | 27 | 0 | 25 | 27 | +| Jev, BM25 top-20 then decide | 160 | 57.4% | 62.0% | 70.5% | 1 | 21† | 1542 | 3598 | +| Jev, embedding top-20 then decide | 160 | 62.0% | 66.7% | 86.8% | 1 | 5 | 1527 | 2611 | +| Jev only, family then decide (BM25 cut for >254) | 160 | 62.0–64.3% | 67.4–69.0% | 70.5% | 1 | 0–2 | 1275 | 2138 | +| Jev, family then decide, embedding cut for >254 (**product default**) | 160 | 62.8% | 67.4% | 86.8% | 1 | 4 | 1287 | 2018 | + +† the 3 s per-evaluation deadline of an earlier build; raised to 6 s in the +product and 20 s in the bench, after which errors are the residual proxy +timeouts shown on the other rows. + +## Composio actions — the heavy catalogue + +| ranker | labelled | top-1 | top-3 | retriever recall@20 | +|---|---|---|---|---| +| bm25 | 66 | 18.2% | 36.4% | 72.7% | +| overlap | 66 | 34.8% | 50.0% | 68.2% | +| Jev, BM25 top-20 | 66 | 66.7% | 72.7% | 72.7% | +| Jev, embedding top-20 | 66 | 74.2% | 78.8% | 90.9% | +| Jev only, family then decide | 66 | 77.3–83.3% | 86.4–90.9% | 72.7% | +| Jev, family then decide + embedding cut | 66 | 80.3% | 87.9% | 90.9% | + +Ranges are two runs of the same configuration: Jev's answers vary by a few +points run to run. + +What the rows say: + +- **Retrieval was the ceiling.** With BM25 shortlisting, Jev's Composio top-3 + (72.7%) equals BM25's recall@20 (72.7%): Jev picked correctly from + everything it was shown. A paraphrase ("ping alex" → `SLACK_SEND_MESSAGE`) + never reached it. +- **Letting Jev pick the family first removes the shortlist for every toolkit + that fits one choice** (all but GitHub's 500 actions), and Composio top-3 + goes to 86–91%. The remaining misses are near-synonyms + (`NOTION_APPEND_TEXT_BLOCKS` for `NOTION_ADD_PAGE_CONTENT`, + `INSTAGRAM_GET_IG_MEDIA_COMMENTS` for `INSTAGRAM_GET_POST_COMMENTS`) and + GitHub actions the BM25 cut dropped. +- **Embeddings replace the lexical cut** for a family larger than one choice + and lift recall@20 to 90.9%; that is the product default: + `FamilyThenDecide` with `EmbeddingToolRanker` as the retriever, BM25 only + when the process has no embedder. Catalogue embeddings are computed once + per process (batches of 64) and cached on disk under + `/cache/tool_search_embeddings.json`. +- **Needless calls collapse**: 26/31 tool-less requests got a BM25 hit; every + Jev configuration answers at most one, because Jev's `none` option and + `needs_tool` abstain. +- **Core tools score lower under Jev than Composio** (46–54% top-3) because + `tinytools-jev` now abstains when `needs_tool < 0.5` or `none` beats the + best option, and many core-tool intents ("what did I tell you about my + dog?", "show me my todos") read as answerable without a tool. In the + product these tools are `Direct` — on the wire, never searched — so the + Composio column is the one `tool_search` is measured by. +- **Latency** is 1.3 s p50 for the family strategy (two proxy round trips, + the second stage's families evaluated concurrently), against a sub-agent + run of several model calls. + +## Reproducing + +```text +cargo run -p openhuman-cli --bin tool-search-bench -- --ranker all --misses +cargo run -p openhuman-cli --bin tool-search-bench -- --ranker jev --family --embedding +cargo run -p openhuman-cli --bin tool-search-bench -- --dump-catalogue +``` + +`OPENHUMAN_BACKEND_API_KEY` (or `TYPESAFE_API_KEY`) selects the key; without +one the bench ranks through the signed-in TinyHumans session exactly as the +product does. From b8b5a0c160dd92e50d57b8b875da5402f88dca8e Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 22 Sep 2026 13:03:51 +0530 Subject: [PATCH 12/14] feat(ranker): switch default strategy to RetrieveThenDecide and fail on missing embeddings The default Jev ranker strategy changes from FamilyThenDecide to RetrieveThenDecide, using the embedding provider to retrieve the top 20 tools by meaning before a single Jev evaluation with a 6-second deadline. When no usable embedding provider is available, the retriever now returns an error instead of falling back to BM25, because a lexical shortlist would cap Jev at BM25's recall (measured at 70% on the Composio catalogue) while adding a network round trip. A new test verifies that a `none` embedding provider disables the search outright. Auto-committed-on: macbook --- crates/openhuman-tinyhumans/src/jev/ranker.rs | 35 ++++++++++++------- .../src/jev/ranker_tests.rs | 21 +++++++++++ 2 files changed, 43 insertions(+), 13 deletions(-) diff --git a/crates/openhuman-tinyhumans/src/jev/ranker.rs b/crates/openhuman-tinyhumans/src/jev/ranker.rs index 33c6236261..4c020c5913 100644 --- a/crates/openhuman-tinyhumans/src/jev/ranker.rs +++ b/crates/openhuman-tinyhumans/src/jev/ranker.rs @@ -66,12 +66,14 @@ impl Default for TinyHumansJevRanker { } impl TinyHumansJevRanker { - /// A ranker with the product defaults: family-then-decide (the - /// evaluator picks the toolkit or pack, then the tool), the process's - /// embedding provider as the retriever for any family too large for one - /// choice, a 3 s deadline per evaluation. + /// A ranker with the product defaults: the process's embedding provider + /// retrieves the top 20 tools by meaning, one Jev evaluation decides + /// (`RetrieveThenDecide`), 6 s deadline per evaluation. Without a usable + /// embedding provider the search does not run and the harness ranks + /// with BM25 alone — a lexical shortlist would cap Jev at BM25's recall, + /// which the bench measured at 70% on the Composio catalogue. pub fn new() -> Self { - Self::with_config(JevRankerConfig::new().with_strategy(JevStrategy::FamilyThenDecide)) + Self::with_config(JevRankerConfig::new().with_strategy(JevStrategy::RetrieveThenDecide)) } /// A ranker with an explicit `tinytools-jev` configuration. @@ -139,7 +141,7 @@ impl TinyHumansJevRanker { // across rebuilds so the catalogue is embedded once per process. let retriever: Arc = match cached.as_ref() { Some(entry) => entry.retriever.clone(), - None => retriever_for(&config), + None => retriever_for(&config)?, }; let ranker = JevRanker::new( evaluator, @@ -163,32 +165,39 @@ impl TinyHumansJevRanker { } } -/// The semantic retriever for `config`'s embedding provider, or BM25 when -/// the provider cannot embed (`none`, or a managed provider with no route). -fn retriever_for(config: &Config) -> Arc { +/// The semantic retriever for `config`'s embedding provider. +/// +/// A provider that cannot embed (`none`) is an error, not a BM25 substitute: +/// the harness answers the search with its own BM25 catalogue in that case, +/// and a Jev decision over a lexical shortlist would only add a network +/// round trip to the same recall. +fn retriever_for(config: &Config) -> Result, RankError> { let provider = openhuman_core::inference::embedding_host::default_embedding_provider_with_config( config, ); if !EmbeddingToolRanker::provider_is_usable(provider.as_ref()) { log::info!( - "[tool-search] embedding provider `{}` cannot embed; retrieving with bm25", + "[tool-search] embedding provider `{}` cannot embed; jev search disabled, bm25 answers", provider.name() ); - return Arc::new(tinytools::Bm25Ranker); + return Err(RankError::backend(format!( + "no usable embedding provider (`{}`); jev search disabled", + provider.name() + ))); } log::info!( "[tool-search] retrieving with embeddings ({} / {})", provider.name(), provider.model_id() ); - Arc::new( + Ok(Arc::new( EmbeddingToolRanker::new(provider).with_disk_cache( config .workspace_dir .join("cache") .join("tool_search_embeddings.json"), ), - ) + )) } fn fingerprint(secret: &str, base_url: &str) -> u64 { diff --git a/crates/openhuman-tinyhumans/src/jev/ranker_tests.rs b/crates/openhuman-tinyhumans/src/jev/ranker_tests.rs index b3b7501e57..d91995a832 100644 --- a/crates/openhuman-tinyhumans/src/jev/ranker_tests.rs +++ b/crates/openhuman-tinyhumans/src/jev/ranker_tests.rs @@ -51,3 +51,24 @@ async fn no_credential_is_a_backend_error_naming_the_gap() { other => panic!("expected a backend error, got {other}"), } } + +/// A config whose embedding provider is `none` disables the Jev search +/// outright — the harness's BM25 answers — rather than quietly retrieving +/// lexically. +#[test] +fn no_embedding_provider_disables_the_search() { + let tmp = tempfile::tempdir().expect("tempdir"); + let mut config = Config { + workspace_dir: tmp.path().join("workspace"), + action_dir: tmp.path().join("workspace"), + config_path: tmp.path().join("config.toml"), + ..Config::default() + }; + config.memory.embedding_provider = "none".into(); + let err = retriever_for(&config).err().map(|e| e.to_string()); + assert!( + err.as_deref() + .is_some_and(|e| e.contains("no usable embedding provider")), + "{err:?}" + ); +} From 2eb5219d9b4404d2ea1ba7ecfb0ef9926ae5bfbb Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 22 Sep 2026 13:04:14 +0530 Subject: [PATCH 13/14] feat(jev): default to embedding top-20 then decide; no embedder, no Jev search The product ranker now retrieves the top 20 tools by meaning through the process's embedding provider and decides with one Jev evaluation, one proxy round trip. Without a usable embedding provider the ranker returns an error and the harness's BM25 bridge answers alone: a Jev decision over a lexical shortlist was measured to add a round trip and nothing else. Co-authored-by: Medulla --- docs/plans/jev-tool-search-baseline.md | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/docs/plans/jev-tool-search-baseline.md b/docs/plans/jev-tool-search-baseline.md index 0b556c01e3..578d1bb23b 100644 --- a/docs/plans/jev-tool-search-baseline.md +++ b/docs/plans/jev-tool-search-baseline.md @@ -24,9 +24,9 @@ returns; no sub-agent. | bm25 | 160 | 22.5% | 38.0% | 70.5% | 26 | 0 | 28 | 29 | | overlap (`rank_tools_by_prompt`, the sub-agent's narrowing today) | 160 | 35.7% | 52.7% | 69.0% | 27 | 0 | 25 | 27 | | Jev, BM25 top-20 then decide | 160 | 57.4% | 62.0% | 70.5% | 1 | 21† | 1542 | 3598 | -| Jev, embedding top-20 then decide | 160 | 62.0% | 66.7% | 86.8% | 1 | 5 | 1527 | 2611 | +| Jev, embedding top-20 then decide (**product default**) | 160 | 62.0% | 66.7% | 86.8% | 1 | 5 | 1527 | 2611 | | Jev only, family then decide (BM25 cut for >254) | 160 | 62.0–64.3% | 67.4–69.0% | 70.5% | 1 | 0–2 | 1275 | 2138 | -| Jev, family then decide, embedding cut for >254 (**product default**) | 160 | 62.8% | 67.4% | 86.8% | 1 | 4 | 1287 | 2018 | +| Jev, family then decide, embedding cut for >254 | 160 | 62.8% | 67.4% | 86.8% | 1 | 4 | 1287 | 2018 | † the 3 s per-evaluation deadline of an earlier build; raised to 6 s in the product and 20 s in the bench, after which errors are the residual proxy @@ -58,12 +58,18 @@ What the rows say: (`NOTION_APPEND_TEXT_BLOCKS` for `NOTION_ADD_PAGE_CONTENT`, `INSTAGRAM_GET_IG_MEDIA_COMMENTS` for `INSTAGRAM_GET_POST_COMMENTS`) and GitHub actions the BM25 cut dropped. -- **Embeddings replace the lexical cut** for a family larger than one choice - and lift recall@20 to 90.9%; that is the product default: - `FamilyThenDecide` with `EmbeddingToolRanker` as the retriever, BM25 only - when the process has no embedder. Catalogue embeddings are computed once - per process (batches of 64) and cached on disk under - `/cache/tool_search_embeddings.json`. +- **Embeddings lift recall@20 to 90.9%** on Composio, and that retriever + with one Jev decision is the product default: `RetrieveThenDecide` over + `EmbeddingToolRanker`, one proxy round trip. Family-then-decide scores a + few points higher on Composio at a second round trip and stays available + through `JevRankerConfig::with_strategy`. Catalogue embeddings are computed + once per process (19 batches of 64 for this catalogue) and cached on disk + under `/cache/tool_search_embeddings.json`, keyed by the + provider's signature; every later search embeds only the intent. +- **No embedder, no Jev search.** When the configured embedding provider is + `none`, `TinyHumansJevRanker` returns an error and the harness answers with + its own BM25 catalogue — a Jev decision over a lexical shortlist would only + add a round trip to the same recall. - **Needless calls collapse**: 26/31 tool-less requests got a BM25 hit; every Jev configuration answers at most one, because Jev's `none` option and `needs_tool` abstain. From 9495412e8be2dea747f8a4a7325ab1a23fa10104 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 22 Sep 2026 13:07:18 +0530 Subject: [PATCH 14/14] test(embedding-ranker): add incremental embedding test for new or changed tools Add a test that verifies the embedding ranker only embeds new or changed tool descriptions incrementally, rather than re-embedding the entire catalogue, when a new toolkit is connected or an existing tool's description is rewritten. Auto-committed-on: macbook --- .../discovery/embedding_ranker_tests.rs | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/crates/openhuman-core/src/agent/tinyagents/discovery/embedding_ranker_tests.rs b/crates/openhuman-core/src/agent/tinyagents/discovery/embedding_ranker_tests.rs index 5243f927b6..d0ab881fdd 100644 --- a/crates/openhuman-core/src/agent/tinyagents/discovery/embedding_ranker_tests.rs +++ b/crates/openhuman-core/src/agent/tinyagents/discovery/embedding_ranker_tests.rs @@ -118,3 +118,34 @@ async fn empty_intent_is_rejected_and_none_provider_is_unusable() { ); assert!(!EmbeddingToolRanker::provider_is_usable(&none)); } + +/// A tool that appears later — a newly connected toolkit's actions, a +/// rewritten description — is embedded on its own; the rest is a cache hit. +#[tokio::test] +async fn a_new_or_changed_tool_is_embedded_incrementally() { + let embedder = Arc::new(BagEmbedder { + calls: AtomicUsize::new(0), + }); + let ranker = EmbeddingToolRanker::new(embedder.clone()); + ranker + .rank("ping", &RankContext::empty(), &candidates(), 1) + .await + .unwrap(); + assert_eq!(embedder.calls.load(Ordering::SeqCst), 2, "catalogue + intent"); + + let mut grown = candidates(); + grown.push(RankCandidate::new("NOTION_CREATE_PAGE", "create a page").with_family("notion")); + grown[2] = RankCandidate::new("file_read", "read a file from disk"); + ranker + .rank("ping", &RankContext::empty(), &grown, 1) + .await + .unwrap(); + // One batch for the two unseen texts (the new tool and the changed one), + // plus the intent — never the whole catalogue again. + assert_eq!(embedder.calls.load(Ordering::SeqCst), 4); + assert_eq!( + ranker.cache.read().unwrap().len(), + 5, + "old and new descriptions both cached; a stale entry is harmless" + ); +}