diff --git a/.env.example b/.env.example index f8a63e24..51e1e246 100644 --- a/.env.example +++ b/.env.example @@ -125,7 +125,7 @@ OPENAI_API_KEY= # # OPENAI_BASE_URL=https://gateway.internal/v1 # OPENAI_API_KEY=... -# BOT_MODEL=openai/gpt-5.6-terra +# BOT_MODEL=openai/gpt-5.5 # # OPENAI_BASE_URL= @@ -141,9 +141,11 @@ OPENAI_API_KEY= # ANTHROPIC_API_KEY= # GOOGLE_API_KEY= -# Which model. Defaults per provider: gpt-5.6-terra, claude-sonnet-4-5, gemini-2.5-flash. -# OpenAI's 5.6 tiers are sol (most capable), terra (the default here) and luna (cheapest). -# BOT_MODEL=gpt-5.6-terra +# Which model the framework Bot uses. Defaults per provider: gpt-5.5, claude-sonnet-4-5, +# gemini-2.5-flash. Not a 5.6 tier: this integration answers nothing at all on gpt-5.6-* through the +# Responses API, driven against the real service. Set one here to try it and the Responses API is +# switched on automatically. The built-in Bots do run 5.6, through the package's model.yaml. +# BOT_MODEL=gpt-5.5 # OpenAI only, and rarely needed: the framework Bot turns the Responses API on by itself for models # that require it. Set it when you are using a model this build has not heard of that needs it too. @@ -238,10 +240,13 @@ MANAGED_AGENT_TOKEN= # proof of concept, and is reached the same way: point MANAGED_AGENT_AG_UI_URL at it, or add it as a # Bot of its own in the tenant package or at /agents. -# Which model the Bots use. agent-langgraph runs gpt-5.6-terra and switches to the Responses API by -# itself, because 5.6 rejects function tools on /v1/chat/completions. agent-bot speaks that endpoint -# by hand and stays on gpt-5.5: the alternative there is reasoning_effort 'none', and a Bot that has -# to decide when to ask a person for help should not be the one with its reasoning turned off. +# Which model the Bots use. BOT_MODEL is the framework Bot's: it runs gpt-5.6-terra and switches to +# the Responses API by itself, because 5.6 rejects function tools on /v1/chat/completions. +# +# The proof-of-concept Bot has its own, AGENT_BOT_MODEL, defaulting to gpt-5.5, because it writes +# that endpoint by hand and refuses to start on a model whose tools it cannot use. One variable for +# both would mean setting the framework Bot's model quietly took the other one's tools away. +# AGENT_BOT_MODEL=gpt-5.5 # BOT_RESPONSES_API=false # One computer per Bot. Unset, every Bot shares the computer at AGENT_COMPUTER_URL, suitable on a diff --git a/CHANGELOG.md b/CHANGELOG.md index cdc8f316..d846c95a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -235,14 +235,18 @@ Sessions survive and nobody signs in again. shipped `gpt-4.1` as the default for every built-in Bot. Asked to open a page behind a sign-in, those Bots answered "would you like me to prompt you to sign in?" and called nothing, three times out of three, while the prompt forbids that sentence in as many words. On `gpt-5.6-terra` the same - question produces the tool call first try. The default is now `gpt-5.6-terra` across the package, - the compose services and both example Bots, and the Responses API is inferred from the model rather - than left to a separate switch, because `gpt-5.6-*` rejects function tools on chat completions and - a deployment that set the model without knowing that got a Bot which started, looked healthy, and - failed on its first tool call. It is a default, not a commitment: `BOT_MODEL` and the package's - `model.yaml` still decide. `agent-bot` stays on `gpt-5.5` on purpose, since the only ways to 5.6 on - the endpoint it writes by hand are a streaming rewrite or turning reasoning off, and it is the Bot - whose job includes deciding when to ask a person for help. + question produces the tool call first try, so the package now runs `gpt-5.6-terra`. It is a + default, not a commitment: `model.yaml` still decides. + + The Bots that answer over AG-UI stay on `gpt-5.5`, each for its own measured reason. The framework + Bot answers nothing at all on `gpt-5.6-*` through the Responses API — `RUN_STARTED`, then + `RUN_FINISHED`, no text — and the hand-written one cannot use function tools on + `/v1/chat/completions` with a 5.6 model unless reasoning is turned off, which is the wrong trade + for a Bot whose job includes deciding when to ask a person for help. It refuses to start on such a + model now rather than failing one silent tool call at a time. Where a 5.6 model is set deliberately, + the Responses API is switched on for it automatically, because a deployment that set the model and + did not know about that switch got a Bot which started, looked healthy, and failed on its first + tool call. - **A Bot browsed to a vendor this deployment already connects to.** A Bot holding no grants was told nothing about connectors at all, so it treated a connected vendor as an ordinary website: asked about Google Drive it opened `drive.google.com`, met a sign-in page, and asked the person to sign diff --git a/agent-bot/src/history.ts b/agent-bot/src/history.ts index 5a3e4920..b4f09a68 100644 --- a/agent-bot/src/history.ts +++ b/agent-bot/src/history.ts @@ -32,7 +32,25 @@ export function toProviderMessages( .filter((id): id is string => Boolean(id)), ); + /* + * Tool results, by the call they answer. + * + * The history is not guaranteed to arrive with a result after the call it belongs to. Read back + * from the durable thread store it arrives the other way round, result first, which is a payload + * no provider accepts: a tool message with no preceding call, and then a call with nothing + * following it. The model answers that with silence rather than an error, which is the worst of + * both, so the pairing is rebuilt here instead of trusted. + */ + const resultsByCall = new Map(); for (const message of input.messages) { + if (message.role !== "tool") continue; + const id = (message as { toolCallId?: string }).toolCallId; + if (id) resultsByCall.set(id, String(message.content ?? "")); + } + + for (const message of input.messages) { + // Placed with the call they answer, below, rather than wherever they arrived. + if (message.role === "tool") continue; if (message.role === "user") { messages.push({ role: "user", content: String(message.content ?? "") }); continue; @@ -41,22 +59,19 @@ export function toProviderMessages( messages.push({ role: "system", content: String(message.content ?? "") }); continue; } - if (message.role === "tool") { - // Tool results are appended so the model can continue from the completed call. - messages.push({ - role: "tool", - tool_call_id: message.toolCallId, - content: String(message.content ?? ""), - }); - continue; - } if (message.role === "assistant") { const toolCalls = message.toolCalls?.map((call) => ({ id: call.id, type: "function" as const, function: { - name: call.function.name, - arguments: call.function.arguments, + /* + * A name is required by the provider and is not always present: read back from the thread + * store these arrive undefined, and a payload carrying `"name": undefined` is rejected + * outright. The call still has to be shown, or the model repeats an action it already + * took, so it keeps its id and is named as something the model can read. + */ + name: call.function?.name ?? "tool", + arguments: call.function?.arguments ?? "{}", }, })); messages.push({ @@ -72,14 +87,22 @@ export function toProviderMessages( * call, so these go here rather than being appended at the end. A call answered later in the * history is left alone and its real answer arrives in its own turn. */ + /* + * Every call this message made, answered, immediately after it. + * + * The real result where there is one, wherever it arrived in the input, and `NO_ANSWER_CAME` + * where there is not. Both cases are the same requirement: a call must be followed by its + * result, and the provider rejects the message outright otherwise. + */ for (const call of message.toolCalls ?? []) { - if (call.id && !answered.has(call.id)) { - messages.push({ - role: "tool", - tool_call_id: call.id, - content: NO_ANSWER_CAME, - }); - } + if (!call.id) continue; + messages.push({ + role: "tool", + tool_call_id: call.id, + content: answered.has(call.id) + ? (resultsByCall.get(call.id) ?? "") + : NO_ANSWER_CAME, + }); } } } diff --git a/agent-bot/src/index.ts b/agent-bot/src/index.ts index da4d18a0..b05cf02d 100644 --- a/agent-bot/src/index.ts +++ b/agent-bot/src/index.ts @@ -33,6 +33,25 @@ if (!MANAGED_AGENT_TOKEN) { * chat-completions streaming loop. */ const MODEL = process.env.BOT_MODEL ?? "gpt-5.5"; +/* + * Refuse a model this file cannot use, rather than discover it one tool call at a time. + * + * `gpt-5.6-*` rejects function tools on `/v1/chat/completions`: "To use function tools, use + * /v1/responses or set reasoning_effort to 'none'." The provider answers with an error, this Bot + * ends the run, and the person sees no reply and no reason. Silence is the worst failure available + * here, and it is what a single mistaken `BOT_MODEL` produced: every tool-using turn stopped dead + * while the Bot looked healthy. + * + * Startup is where a deployment can act on it, which is the same posture as the token check above. + */ +if (/^gpt-5\.[6-9]|^gpt-[6-9]/.test(MODEL)) { + console.error( + `BOT_MODEL=${MODEL} cannot be used by this Bot. It speaks /v1/chat/completions directly, and ` + + "that endpoint refuses function tools for this model, so every tool call would fail with no " + + "reply. Use gpt-5.5, or the framework Bot on port 4201, which speaks the Responses API.", + ); + process.exit(1); +} /** * Where that model is answered from. diff --git a/agent-bot/tests/history.test.ts b/agent-bot/tests/history.test.ts index 1318c638..540d29d8 100644 --- a/agent-bot/tests/history.test.ts +++ b/agent-bot/tests/history.test.ts @@ -146,3 +146,64 @@ describe("a tool call nothing ever answered", () => { expect(ids).toEqual(["c1", "c2"]); }); }); + +/** + * The history as the durable thread store hands it back. + * + * Read back from a stored thread, a tool result arrives BEFORE the assistant message that made the + * call, and the call's `function.name` is missing. Both are payloads a provider rejects: a tool + * message with no preceding call, and a call with nothing following it. The model answers that with + * silence rather than an error, so a Bot that had just read a document said nothing at all and the + * conversation looked dead. + * + * The exact shape below was copied off a real thread after a Google Drive answer went missing. + */ +describe("a history that arrives out of order", () => { + test("pairs each call with its result, whatever order they arrived in", () => { + const messages = withoutGuidance( + toProviderMessages( + input([ + { id: "1", role: "user", content: "What is in the PRD?" } as Message, + { + id: "2", + role: "tool", + toolCallId: "c1", + content: "the document text", + } as unknown as Message, + { + id: "3", + role: "assistant", + content: "", + toolCalls: [call("c1", "read_file_content")], + } as unknown as Message, + ]), + ), + ); + + // Assistant first, then its result. Never a tool message with no call before it. + expect(messages.map((m) => m.role)).toEqual(["user", "assistant", "tool"]); + const answer = messages[2] as { tool_call_id?: string; content?: string }; + expect(answer.tool_call_id).toBe("c1"); + expect(answer.content).toBe("the document text"); + }); + + test("gives a nameless call a name, because the provider requires one", () => { + const messages = withoutGuidance( + toProviderMessages( + input([ + { + id: "1", + role: "assistant", + content: "", + toolCalls: [{ id: "c1", type: "function", function: {} }], + } as unknown as Message, + ]), + ), + ); + + const assistant = messages[0] as { + tool_calls?: { function: { name: string } }[]; + }; + expect(assistant.tool_calls?.[0]?.function.name).toBe("tool"); + }); +}); diff --git a/agent-langgraph/src/index.ts b/agent-langgraph/src/index.ts index b8a697d6..b1302a46 100644 --- a/agent-langgraph/src/index.ts +++ b/agent-langgraph/src/index.ts @@ -100,7 +100,7 @@ const GOOGLE_BASE_URL = function defaultModelFor(provider: string): string { if (provider === "anthropic") return "claude-sonnet-4-5"; if (provider === "google") return "gemini-2.5-flash"; - return "gpt-5.6-terra"; + return "gpt-5.5"; } /** diff --git a/app/src/components/channels/channel-chat.tsx b/app/src/components/channels/channel-chat.tsx index 89148a5a..0f4a4e25 100644 --- a/app/src/components/channels/channel-chat.tsx +++ b/app/src/components/channels/channel-chat.tsx @@ -6,7 +6,6 @@ import { } from "@copilotkit/react-core/v2"; import { useMutation, useQuery } from "@tanstack/react-query"; import { useCallback, useEffect, useRef, useState } from "react"; -import { readThreadMessages } from "@/lib/copilot/thread-messages"; import { toAgentOptions } from "@/components/channels/composer"; import { ConversationView } from "@/components/channels/conversation-view"; import { @@ -22,6 +21,7 @@ import { ConversationProvider } from "@/lib/copilot/conversation"; import { afterMs, joinWithin } from "@/lib/copilot/join-thread"; import { repairUnansweredToolCalls } from "@/lib/copilot/repair-history"; import { stoppedReason } from "@/lib/copilot/stopped-turn"; +import { readThreadMessages } from "@/lib/copilot/thread-messages"; import { useSkillCommands } from "@/lib/plugins/skill-commands"; import { newId } from "../../lib/new-id"; @@ -125,12 +125,20 @@ export function ChannelChat({ let current = true; void (async () => { - // Bounded, and finished when it returns; `join-thread.ts` has why that matters. - await joinWithin({ - connect: copilotkit.connectAgent({ agent }), - deadline: afterMs(JOIN_DEADLINE_MS), - detach: () => agent.detachActiveRun(), - }); + try { + // Bounded, and finished when it returns; `join-thread.ts` has why that matters. + await joinWithin({ + connect: copilotkit.connectAgent({ agent }), + deadline: afterMs(JOIN_DEADLINE_MS), + detach: () => agent.detachActiveRun(), + }); + } catch { + /* + * A join that throws is a join that is over. It must not take the gate with it: everything + * typed afterwards waits on that gate, so a throw here would silence the conversation + * rather than degrade it. History is restored below either way. + */ + } try { const stored = await readThreadMessages( diff --git a/app/src/lib/copilot/join-thread.ts b/app/src/lib/copilot/join-thread.ts index f717cd12..028cccd2 100644 --- a/app/src/lib/copilot/join-thread.ts +++ b/app/src/lib/copilot/join-thread.ts @@ -39,9 +39,30 @@ export async function joinWithin({ // A detach with nothing to detach is not a problem worth reporting, and the wait below is what // this function actually promises. Swallowing it here keeps that promise on both paths. } - await finished; + /* + * Bounded, because a detach is a request and not a guarantee. + * + * This was a bare `await finished`, on the reasoning that a detached connect ends promptly. When + * it does not, nothing here ever returns: the caller's `finally` never runs, the gate it opens + * stays shut, and every message typed afterwards waits on it forever. That is silence — the + * message appears in the transcript, no run is ever started, no request reaches the server and + * nothing is logged, which is the hardest failure of all to read. + * + * A connect still running after this grace has outlived its usefulness either way. Going on + * without it risks the overwrite this function exists to prevent; waiting for it risks a + * conversation that never answers again. The first is recoverable and visible. The second is not. + */ + await Promise.race([finished, afterMs(DETACH_GRACE_MS)]); } +/** + * How long a detached connect is given to finish before the turn goes ahead regardless. + * + * Long enough that an ending connect is waited for, short enough that a stuck one is not the end of + * the conversation. + */ +const DETACH_GRACE_MS = 2_000; + /** A deadline, as a promise. Separate so a test can supply one it controls. */ export function afterMs(milliseconds: number): Promise { return new Promise((resolve) => setTimeout(resolve, milliseconds)); diff --git a/app/tests/join-thread.test.ts b/app/tests/join-thread.test.ts index 32ed69c4..09d50a0c 100644 --- a/app/tests/join-thread.test.ts +++ b/app/tests/join-thread.test.ts @@ -121,3 +121,61 @@ describe("joinWithin", () => { expect(done).toBe(true); }); }); + +/** + * A connect that does not end when it is asked to. + * + * `detach` is a request, not a guarantee. The wait after it used to be unbounded, on the reasoning + * that a detached connect ends promptly, and when it does not nothing here ever returns: the + * caller's `finally` never runs, the gate it opens stays shut, and every message typed afterwards + * waits on it forever. + * + * That failure is silent. The message appears in the transcript, no run is started, no request + * reaches the server, and nothing is logged. Two Bots sat mute through a whole gate run before this + * was found, and the only symptom was an answer that never came. + */ +describe("a detached connect that never finishes", () => { + test("does not hold the turn forever", async () => { + let ended = false; + const never = new Promise(() => {}); + + const settled = joinWithin({ + connect: never, + deadline: Promise.resolve(), + detach: async () => { + // Asked, and ignored, which is the case this exists for. + }, + }).then(() => { + ended = true; + }); + + await Promise.race([ + settled, + new Promise((resolve) => setTimeout(resolve, 4_000)), + ]); + + expect(ended).toBe(true); + }, 10_000); + + test("still waits for a detached connect that does finish", async () => { + // The behaviour the grace must not throw away: a connect that ends is waited for, so nothing is + // left in flight to overwrite the message. + let finished = false; + let end: () => void = () => {}; + const connect = new Promise((resolve) => { + end = () => { + finished = true; + resolve(); + }; + }); + + const settled = joinWithin({ + connect, + deadline: Promise.resolve(), + detach: async () => end(), + }); + + await settled; + expect(finished).toBe(true); + }); +}); diff --git a/docker-compose.yml b/docker-compose.yml index bbe2a643..6fcd2a52 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -177,7 +177,10 @@ services: # Unset means OpenAI. Set, it is any endpoint speaking the same API, and BOT_MODEL is sent # to it verbatim. OPENAI_BASE_URL: ${OPENAI_BASE_URL:-} - BOT_MODEL: ${BOT_MODEL:-gpt-5.6-terra} + # gpt-5.5, not the deployment default: this Bot writes /v1/chat/completions by hand, and + # gpt-5.6-* rejects function tools there unless reasoning is turned off. Its own variable, so + # a BOT_MODEL set for the framework Bot cannot silently take its tools away. + BOT_MODEL: ${AGENT_BOT_MODEL:-gpt-5.5} healthcheck: test: ["CMD-SHELL", "bun -e \"await fetch('http://localhost:4200/health')\""] interval: 10s @@ -204,7 +207,10 @@ services: ANTHROPIC_BASE_URL: ${ANTHROPIC_BASE_URL:-} GOOGLE_API_KEY: ${GOOGLE_API_KEY:-} GOOGLE_GENERATIVE_AI_BASE_URL: ${GOOGLE_GENERATIVE_AI_BASE_URL:-} - BOT_MODEL: ${BOT_MODEL:-gpt-5.6-terra} + # gpt-5.5. This integration answers nothing at all on gpt-5.6-* through the Responses API: + # RUN_STARTED, then RUN_FINISHED, no text. Driven against the real service. 5.6 is still + # reachable by setting BOT_MODEL, and the Responses API is switched on for it automatically. + BOT_MODEL: ${BOT_MODEL:-gpt-5.5} BOT_RESPONSES_API: ${BOT_RESPONSES_API:-false} # Where this Bot runs a tool: back through the deployment that granted it, never at the vendor. # `host.docker.internal` because the API server runs on the host, not in this network. diff --git a/examples/langgraph-bot/src/index.ts b/examples/langgraph-bot/src/index.ts index 5a96f544..e8308cc1 100644 --- a/examples/langgraph-bot/src/index.ts +++ b/examples/langgraph-bot/src/index.ts @@ -27,7 +27,7 @@ import { serve } from "bun"; */ const PORT = Number.parseInt(process.env.PORT ?? "4300", 10); -const MODEL = process.env.BOT_MODEL ?? "gpt-5.6-terra"; +const MODEL = process.env.BOT_MODEL ?? "gpt-5.5"; /** * `gpt-5.6-*` rejects function tools on `/v1/chat/completions` and needs the Responses API, which * this integration speaks. Inferred from the model so setting `BOT_MODEL` alone cannot produce a diff --git a/examples/mastra-bot/src/index.ts b/examples/mastra-bot/src/index.ts index 0d99f3e8..c39b79c3 100644 --- a/examples/mastra-bot/src/index.ts +++ b/examples/mastra-bot/src/index.ts @@ -17,7 +17,7 @@ import { serve } from "bun"; */ const PORT = Number.parseInt(process.env.PORT ?? "4400", 10); -const MODEL = process.env.BOT_MODEL ?? "gpt-5.6-terra"; +const MODEL = process.env.BOT_MODEL ?? "gpt-5.5"; /** * `gpt-5.6-*` rejects function tools on `/v1/chat/completions` and needs the Responses API, which * this provider exposes as `openai.responses`. Inferred from the model so setting `BOT_MODEL` alone diff --git a/server/src/routing/classify.ts b/server/src/routing/classify.ts index 65905c21..5ae457b0 100644 --- a/server/src/routing/classify.ts +++ b/server/src/routing/classify.ts @@ -86,6 +86,47 @@ export function routingPrompt( ].join("\n"); } +/** + * The one coworker that can reach a system this message names, when there is exactly one. + * + * A hint for the router became a decision for the fallback, and only there. A confident match on + * purpose still wins: a specialist with no connectors is the right answer to a question about its + * specialism, and this must not turn reach into a filter that overrides that. + * + * Matched on the system's own id with separators loosened, so `google-drive` answers to "Google + * Drive" as somebody would type it. Deliberately not fuzzy beyond that: a router that guesses at + * near-misses is a router nobody can predict. + */ +function onlyCoworkerReaching( + text: string, + candidates: readonly RoutingCandidate[], +): { id: string; name: string; system: string } | null { + const haystack = text.toLowerCase(); + const named = new Set(); + for (const candidate of candidates) { + for (const system of candidate.reaches ?? []) { + const spelled = system.toLowerCase().replace(/[-_]+/g, " "); + if ( + haystack.includes(spelled) || + haystack.includes(system.toLowerCase()) + ) { + named.add(system); + } + } + } + + for (const system of named) { + const holders = candidates.filter((candidate) => + candidate.reaches?.includes(system), + ); + const only = holders[0]; + if (holders.length === 1 && only) { + return { id: only.id, name: only.name, system }; + } + } + return null; +} + export function createIntentRouter(deps: { /** Runs the prompt and returns the model's raw text. May reject; the router absorbs it. */ complete: (prompt: string) => Promise; @@ -98,6 +139,26 @@ export function createIntentRouter(deps: { ): Promise { const byId = new Map(candidates.map((c) => [c.id, c])); const fallback = (reason: string): RoutingDecision => { + /* + * Before the default, ask whether the message named a system only one coworker can reach. + * + * Every path into here is "we are not sure", and the default is a guess. When the message + * says Google Drive and exactly one coworker holds Google Drive, that is not a guess: the + * others cannot answer it at all, and a Bot with no connector meets a sign-in wall the + * connector exists to avoid. + * + * Only when the answer is unambiguous. Two coworkers holding the same system is a choice + * this cannot make, and it falls through to the default as before. + */ + const reachable = onlyCoworkerReaching(text, candidates); + if (reachable) { + return { + agentId: reachable.id, + name: reachable.name, + reason: `the only coworker that can reach ${reachable.system}`, + fallback: true, + }; + } const chosen = byId.get(defaultId) ?? candidates[0]; return chosen ? { agentId: chosen.id, name: chosen.name, reason, fallback: true } diff --git a/server/src/routing/model.ts b/server/src/routing/model.ts index fc6fa487..1bbb4e41 100644 --- a/server/src/routing/model.ts +++ b/server/src/routing/model.ts @@ -25,7 +25,20 @@ export function createModelCompleter(deps: { }, body: JSON.stringify({ model: deps.model.defaultModel, - temperature: 0, + /* + * No temperature. + * + * It was zero, for a router that answers the same way twice. Reasoning models refuse the + * setting outright — "Unsupported value: 'temperature' does not support 0 with this model. + * Only the default (1) value is supported" — and this call treats a throw as "not sure", so + * every routing decision quietly became the default coworker and the roster was never + * consulted. A question naming Google Drive went to a Bot holding no Drive tools, which is + * the exact failure the roster exists to prevent, and nothing said so. + * + * Omitted rather than set per model, because a list of which models accept it is a list that + * goes stale. `response_format` and a prompt that asks for one object keep the answer tight, + * and the confidence floor still sends an unsure match to the default. + */ response_format: { type: "json_object" }, messages: [{ role: "user", content: prompt }], }), diff --git a/server/tests/routing-classify.test.ts b/server/tests/routing-classify.test.ts index 75a1271f..9407879b 100644 --- a/server/tests/routing-classify.test.ts +++ b/server/tests/routing-classify.test.ts @@ -1,8 +1,8 @@ import { describe, expect, test } from "bun:test"; import { createIntentRouter, - routingPrompt, type RoutingCandidate, + routingPrompt, } from "../src/routing/classify"; const ROSTER: RoutingCandidate[] = [ @@ -160,3 +160,112 @@ describe("routing on what a coworker can reach", () => { expect(plain).not.toContain("can reach"); }); }); + +/** + * A fallback that lands on a coworker who cannot answer. + * + * Every path into the fallback is "we are not sure", and the default is a guess. When the message + * names a system exactly one coworker can reach, it is not a guess: the others cannot answer it at + * all, and a Bot without the connector browses to the vendor and meets a sign-in wall the connector + * exists to avoid. + * + * Found by driving it. The router's model call was throwing on every request, so every decision took + * the unreachable path, and a question naming Google Drive went to a Bot holding no Drive tools. The + * broken call is fixed separately; this is the part that was wrong even when it worked. + */ +describe("falling back to somebody who can actually answer", () => { + const ROSTER: RoutingCandidate[] = [ + { + id: "general-assistant", + name: "General Assistant", + roleDescription: "everyday work", + }, + { + id: "risk-analyst", + name: "Risk Analyst", + roleDescription: "risk and compliance", + reaches: ["google-drive"], + }, + ]; + + const BROKEN = createIntentRouter({ + complete: async () => { + throw new Error("router unreachable"); + }, + }); + + test("names the one coworker that can reach the system in the message", async () => { + const decision = await BROKEN.route( + "In my OpenBot PRD in Google Drive, list the proxy metrics.", + ROSTER, + "general-assistant", + ); + + expect(decision.agentId).toBe("risk-analyst"); + expect(decision.reason).toContain("google-drive"); + // Still a fallback: nothing inferred the fit, the roster just answered it. + expect(decision.fallback).toBe(true); + }); + + test("matches the vendor as somebody would write it", async () => { + // `google-drive` is an id. Nobody types a hyphen. + const decision = await BROKEN.route( + "search google drive for the PRD", + ROSTER, + "general-assistant", + ); + + expect(decision.agentId).toBe("risk-analyst"); + }); + + test("uses the default when the message names no system", async () => { + const decision = await BROKEN.route( + "what is 8 plus 5", + ROSTER, + "general-assistant", + ); + + expect(decision.agentId).toBe("general-assistant"); + }); + + test("uses the default when two coworkers reach the same system", async () => { + // Not a decision this can make. Two holders is exactly the case the router is for. + const shared: RoutingCandidate[] = [ + { ...ROSTER[0], reaches: ["google-drive"] } as RoutingCandidate, + ROSTER[1] as RoutingCandidate, + ]; + + const decision = await BROKEN.route( + "look in google drive", + shared, + "general-assistant", + ); + + expect(decision.agentId).toBe("general-assistant"); + }); + + test("a confident match still wins over reach", async () => { + /* + * The line this must not cross. A specialist with no connectors is the right answer to a + * question about its specialism, and reach is a hint for the router rather than a filter over + * it. This only decides where a guess would otherwise have. + */ + const confident = createIntentRouter({ + complete: async () => + JSON.stringify({ + agentId: "general-assistant", + reason: "everyday work", + confidence: 0.9, + }), + }); + + const decision = await confident.route( + "tidy up my google drive notes", + ROSTER, + "risk-analyst", + ); + + expect(decision.agentId).toBe("general-assistant"); + expect(decision.fallback).toBe(false); + }); +});