+ {DISCOVERY_REASONS[payload.reason] ?? payload.reason}
+ {Array.isArray(payload.skills) && payload.skills.length > 0
+ ? `: ${payload.skills.join(", ")}`
+ : ""}
+
+ ) : null}
{event.eventType === "mcp.callback_refused" &&
typeof payload.refusal === "string" ? (
@@ -332,6 +358,21 @@ function Row({
*
* Anything else falls through to the element or file subject.
*/
+/**
+ * Why a run was offered the tools it was, in words rather than in the slug the server writes.
+ *
+ * Every one of these looks the same from outside: the Bot was handed some tools. The distinction is
+ * the difference between a deployment that narrowed on purpose, one that has never declared a skill,
+ * and one whose selector could not be reached, and only the last is a fault.
+ */
+const DISCOVERY_REASONS: Record
= {
+ "under-floor": "Few enough tools to offer them all",
+ "nothing-declared": "No skill declares any of these tools",
+ unavailable: "Could not choose, so all were offered",
+ "nothing-chosen": "No skill applied, so all were offered",
+ selected: "Chosen by skill",
+};
+
const NAMED_TARGETS = new Set([
"component",
"mcp_tool",
@@ -368,6 +409,9 @@ const DECISIONS: Record = {
// A function failure is execution failure, not a policy refusal.
"component.function_failed": "Could not be read",
+ // Not a call and not a decision: the tools this run was allowed to see. Worded so nobody reads it
+ // as permission, which it is not — everything named was already granted.
+ "mcp.tools_discovered": "Tools offered for one run",
"mcp.call_succeeded": "Called on this Bot's behalf",
"mcp.call_rejected": "Blocked",
"mcp.call_failed": "The server did not answer",
diff --git a/bun.lock b/bun.lock
index 4e6384ee..6a698bc9 100644
--- a/bun.lock
+++ b/bun.lock
@@ -70,6 +70,7 @@
"drizzle-orm": "^0.45.2",
"hono": "^4.10.0",
"postgres": "^3.4.9",
+ "rxjs": "7.8.1",
"yaml": "^2.9.0",
"zod": "^4.4.3",
},
diff --git a/docs/architecture.md b/docs/architecture.md
index 0d50f1c7..d1b521b4 100644
--- a/docs/architecture.md
+++ b/docs/architecture.md
@@ -142,6 +142,14 @@ A catalogue entry says whose credential a Bot reaches it with, which is a differ
Every MCP call checks the grant first, then evaluates the same action policy engine with MCP context, then audits the result.
+### Which tools a run is offered
+
+A model picks the right tool reliably out of about ten, and unreliably out of thirty. A deployment that connects two vendors passes that point on its first afternoon, so a Bot holding more than a handful of tools is offered, per run, only the tools of the skills that match the message.
+
+A skill declares the tools it needs (`skill_tools`). Before the run starts, the deployment asks its own model which skills the message needs, and the Bot is built with those skills' tools plus every granted tool no skill claims. A declaration grants nothing: the offer is always intersected with what the Bot was already granted, so writing a skill can never hand anybody a tool.
+
+This narrows the offer. It is not a boundary, and it never substitutes for one. The grant, the policy and the audit row decide what may happen; this decides only what the model can see. Every way it can fail — no skills declared, a model that cannot answer, a message that matches nothing, twelve tools or fewer — leaves the whole catalogue offered, because a narrowing that failed closed would remove capability an administrator granted, silently. `mcp.tools_discovered` records what was offered, out of how much, and why.
+
## Tenant package and knowledge
`TENANT_PACKAGE_DIR` points at the tenant package. The default is `../examples/fintech`.
diff --git a/server/package.json b/server/package.json
index 768acd46..220afade 100644
--- a/server/package.json
+++ b/server/package.json
@@ -22,6 +22,7 @@
"drizzle-orm": "^0.45.2",
"hono": "^4.10.0",
"postgres": "^3.4.9",
+ "rxjs": "7.8.1",
"yaml": "^2.9.0",
"zod": "^4.4.3"
},
diff --git a/server/src/audit.ts b/server/src/audit.ts
index 3be5e620..2977cec6 100644
--- a/server/src/audit.ts
+++ b/server/src/audit.ts
@@ -65,6 +65,24 @@ export const auditEventTypes = [
* endpoint that dies mid-answer from one that never answers at all.
*/
"agent.stream_stalled",
+ /*
+ * Which of a Bot's tools were put in front of the model for one run, and why those.
+ *
+ * Discovery, recorded as its own fact, because a run is now offered a subset of what the Bot holds
+ * and every other row here answers a question about a call that happened. This one answers "why
+ * did it call that", and its harder twin, "why did it not call anything" — a Bot that had the
+ * right tool granted, was not offered it, and answered from memory leaves no other trace at all.
+ * Without this row that failure is indistinguishable from a model that simply chose badly.
+ *
+ * DISCOVERY IS NOT PERMISSION, and the row is not an authorization record. Everything named here
+ * was already granted; being offered is what changed. A tool still goes through the grant, the
+ * policy and `mcp.call_succeeded` or `mcp.call_rejected` before anything happens, so this row
+ * never appears in place of one of those, only before it.
+ *
+ * `reason` is the part worth reading. It separates a deployment that narrowed from one that never
+ * declared anything and one whose selector was unreachable, which look identical from outside.
+ */
+ "mcp.tools_discovered",
"mcp.call_succeeded",
"mcp.call_rejected",
/*
diff --git a/server/src/copilot.ts b/server/src/copilot.ts
index f316ab52..919ef3b8 100644
--- a/server/src/copilot.ts
+++ b/server/src/copilot.ts
@@ -1,3 +1,4 @@
+import type { BaseEvent, RunAgentInput } from "@ag-ui/client";
import { AbstractAgent, HttpAgent } from "@ag-ui/client";
import type { BuiltInAgentConfiguration } from "@copilotkit/runtime/v2";
import {
@@ -6,6 +7,8 @@ import {
CopilotRuntime,
} from "@copilotkit/runtime/v2";
import { createCopilotHonoHandler } from "@copilotkit/runtime/v2/hono";
+import type { Observable } from "rxjs";
+import { defer, from, switchMap } from "rxjs";
import { z } from "zod";
import {
COMPUTER_GUIDANCE,
@@ -14,6 +17,12 @@ import {
import type { AgentActor } from "./agents/profile-types";
import type { StallGuard } from "./channels/stall-guard";
import type { DeploymentConfig } from "./config";
+import type { SelectableSkill, Selection } from "./plugins/selection";
+import {
+ latestUserText,
+ SELECTION_FLOOR,
+ selectTools,
+} from "./plugins/selection";
import type { GrantedTool } from "./plugins/tools";
import { grantedToolGuidance } from "./plugins/tools";
@@ -285,6 +294,8 @@ export async function buildAgents(
* is a fact about the deployment; what differs per Bot is which of them it holds.
*/
loadVendors: () => Promise = async () => [],
+ /** How a run's tools are narrowed to what it is about. Absent means they are not. */
+ selection?: ToolSelection,
): Promise> {
const vendors = await loadVendors().catch(() => [] as readonly string[]);
return Object.fromEntries(
@@ -300,6 +311,7 @@ export async function buildAgents(
signRun,
computerGuidance,
vendors,
+ selection,
),
]),
),
@@ -315,31 +327,118 @@ async function buildAgent(
signRun?: SignRun,
computerGuidance?: string,
connectedVendors: readonly string[] = [],
+ selection?: ToolSelection,
): Promise {
- if (agent.type === "built_in") {
- return new BuiltInAgent(
+ if (agent.type === "unavailable") {
+ return new UnavailableAgent(agent);
+ }
+
+ const granted = await loadTools(agent.id);
+
+ /*
+ * Whether narrowing can do anything here at all.
+ *
+ * A skill that declares no tools is not a unit of retrieval, and a catalogue already small enough
+ * to choose from has nothing to gain. In both cases the Bot is built exactly as it was before any
+ * of this existed: no deferral, no per-run model call, nothing to go wrong. That is most
+ * deployments on their first day, and they should not pay for a feature they are not using.
+ */
+ const skills = selection
+ ? await selection.loadSkills(agent.id).catch(() => [])
+ : [];
+ const narrowing =
+ selection &&
+ skills.some((skill) => skill.tools.length > 0) &&
+ granted.length > (selection.floor ?? SELECTION_FLOOR)
+ ? selection
+ : undefined;
+
+ /** Pass one and pass two, for one run. Shared by both agent kinds; each applies it differently. */
+ const offeredFor = async (input: RunAgentInput): Promise => {
+ if (!narrowing) return granted;
+ const chosen = await selectTools({
+ tools: granted,
+ skills,
+ text: latestUserText(input.messages),
+ choose: narrowing.choose,
+ ...(narrowing.floor === undefined ? {} : { floor: narrowing.floor }),
+ });
+ // Awaited, so the row is on record before the model is handed the tools it names. A discovery
+ // written afterwards would sit in the trail after the calls it explains.
+ await narrowing.record?.(agent.id, chosen).catch(() => {});
+ return chosen.offered;
+ };
+
+ if (agent.type === "remote_ag_ui") {
+ /*
+ * The remote path narrows inside its own middleware rather than by being wrapped.
+ *
+ * `.use()` middleware is applied by `runAgent`, not by `run`, so an outer agent delegating to
+ * `remote.run(input)` skips it: the endpoint would get a run with no standing role, no holdings
+ * message, no tools and no signed assertion, and every one of those failures is silent.
+ */
+ return remoteAgentWithStandingRole(
+ agent,
+ stallGuard,
+ granted,
+ signRun,
+ connectedVendors,
+ narrowing ? offeredFor : undefined,
+ );
+ }
+
+ /*
+ * A built-in Bot takes its tools in its configuration, so narrowing means building it again once
+ * the message is known. The guidance it is given is generated from the tools passed here, which is
+ * what keeps a narrowed run from being told it holds something it was not offered.
+ */
+ const withTools = (tools: GrantedTool[]) =>
+ new BuiltInAgent(
builtInAgentConfiguration(
agent,
model,
apiKey,
- await loadTools(agent.id),
+ tools,
computerGuidance,
connectedVendors,
),
);
- }
- if (agent.type === "unavailable") {
- return new UnavailableAgent(agent);
- }
- return remoteAgentWithStandingRole(
- agent,
- stallGuard,
- await loadTools(agent.id),
- signRun,
- connectedVendors,
+
+ const whole = withTools(granted);
+ if (!narrowing) return whole;
+
+ return new RunSelectedAgent(
+ { agentId: agent.id, description: agent.name },
+ whole,
+ async (input) => {
+ const offered = await offeredFor(input);
+ // Nothing narrowed means nothing to rebuild, and reusing the agent already built for this
+ // request keeps that path allocation-for-allocation what it was.
+ return offered.length === granted.length ? whole : withTools(offered);
+ },
);
}
+/**
+ * How a deployment narrows a Bot's tools to the ones a run is about. Absent means it does not.
+ *
+ * Three collaborators rather than one, because they fail differently and are configured in
+ * different places: the skills come from the plugin store, the choosing is a model call on the
+ * deployment's own key, and the record goes to the audit trail. A deployment missing any of them
+ * should lose the narrowing and keep the Bot, which is why `record` is optional and the other two
+ * are allowed to throw.
+ */
+export type ToolSelection = {
+ /** What this Bot's granted skills declare. Failure is treated as "no skills". */
+ loadSkills: (botId: string) => Promise;
+ /** Pass one. Returns the model's raw answer; throwing means the narrowing is skipped. */
+ choose: (prompt: string) => Promise;
+ /** Writes the discovery row. Never allowed to fail a run. */
+ record?: (botId: string, selection: Selection) => Promise;
+ /** Overrides the default catalogue size below which nothing is narrowed. */
+ floor?: number;
+};
+
/**
* A remote AG-UI agent that states its standing role on every run.
*
@@ -366,6 +465,19 @@ function remoteAgentWithStandingRole(
signRun?: SignRun,
/** As for the built-in path: what this deployment connects to, held or not. */
connectedVendors: readonly string[] = [],
+ /**
+ * Which of those tools this run is about, decided once the message is known.
+ *
+ * NARROWED HERE RATHER THAN BY WRAPPING THE AGENT, and the difference is not cosmetic. Middleware
+ * registered with `.use()` is applied by `runAgent`, not by `run`: an outer agent that delegated
+ * to `remote.run(input)` would skip this whole function's work, and the endpoint would receive a
+ * run with no standing role, no holdings message, no tools and no signed assertion. Every one of
+ * those is silent — the Bot simply answers worse — so the narrowing goes inside the middleware
+ * that is already here.
+ *
+ * Absent means no narrowing, which is the behaviour every deployment had before this existed.
+ */
+ narrow?: (input: RunAgentInput) => Promise,
) {
const remote = new HttpAgent({
url: agent.endpoint,
@@ -388,18 +500,28 @@ function remoteAgentWithStandingRole(
* tools as an offer and decides for itself what to call, with `COMPUTER_GUIDANCE` as its whole
* prompt — a page about the browser that mentions connectors nowhere. That is the Bot that browsed
* to drive.google.com holding four Drive tools.
+ *
+ * Built from the tools this run was offered rather than from everything granted, so a narrowed
+ * run is never told it holds a system it cannot reach on this turn.
*/
- const holdings = grantedToolGuidance(tools, connectedVendors);
- const holdingsMessage = holdings
- ? {
- id: `granted-tools:${agent.id}`,
- role: "system" as const,
- content: holdings,
- }
- : null;
+ const holdingsMessageFor = (offered: GrantedTool[]) => {
+ const holdings = grantedToolGuidance(offered, connectedVendors);
+ return holdings
+ ? {
+ id: `granted-tools:${agent.id}`,
+ role: "system" as const,
+ content: holdings,
+ }
+ : null;
+ };
- remote.use((input, next) =>
- next.run({
+ const runWith = (
+ tools: GrantedTool[],
+ input: RunAgentInput,
+ next: AbstractAgent,
+ ) => {
+ const holdingsMessage = holdingsMessageFor(tools);
+ return next.run({
...input,
messages: [
agent.standingMessage,
@@ -460,11 +582,110 @@ function remoteAgentWithStandingRole(
*/
{}),
},
- } as never),
+ } as never);
+ };
+
+ /*
+ * Deferred, because choosing the tools is a model call and middleware has to answer with a stream
+ * straight away. `defer` puts the work on the subscription, which is where the run actually
+ * begins, so nothing happens until somebody is listening and a retried run chooses again.
+ */
+ remote.use((input, next) =>
+ defer(() =>
+ from(narrow ? narrow(input) : Promise.resolve(tools)).pipe(
+ switchMap((offered) => runWith(offered, input, next)),
+ ),
+ ),
);
+
return remote;
}
+/**
+ * An agent whose tools are decided when the run starts, because that is the first moment anybody
+ * knows what the run is about.
+ *
+ * WHY A WRAPPER AND NOT A NARROWER `loadTools`. Tools are resolved per request, and a request is
+ * earlier than a run: at that point there is a Bot and a person and no message, so there is nothing
+ * to select against. `run(input)` is the first place the message exists. Both underlying agents take
+ * their tools at construction — a built-in one in its configuration, a remote one in the middleware
+ * that sends them — so the only way to hand either a set chosen from the message is to build it
+ * after the message arrives. That is all this does: it defers `build` to the first subscription and
+ * then gets out of the way.
+ *
+ * The deferral is per subscription, so a retried run reselects rather than reusing a decision made
+ * for a message that is no longer the last one.
+ */
+class RunSelectedAgent extends AbstractAgent {
+ /**
+ * The agent this run turned into, once there is one.
+ *
+ * Held only so `abortRun` can reach it. Without this, pressing stop aborts a wrapper that is not
+ * doing anything and leaves the model call underneath it running to completion, spending the
+ * deployment's money on an answer nobody will see.
+ */
+ private inner?: AbstractAgent;
+ /** The same Bot with nothing narrowed, kept to answer questions that are not about one run. */
+ private whole: AbstractAgent;
+ private build: (input: RunAgentInput) => Promise;
+
+ constructor(
+ identity: { agentId: string; description: string },
+ whole: AbstractAgent,
+ build: (input: RunAgentInput) => Promise,
+ ) {
+ super(identity);
+ this.whole = whole;
+ this.build = build;
+ }
+
+ run(input: RunAgentInput): Observable {
+ return defer(() =>
+ from(this.build(input)).pipe(
+ switchMap((agent) => {
+ this.inner = agent;
+ return agent.run(input);
+ }),
+ ),
+ );
+ }
+
+ /**
+ * What the Bot can do, answered from the un-narrowed agent.
+ *
+ * Capabilities are asked for outside a run, where there is no message and so nothing to select
+ * against. They are also a fact about the Bot rather than about one turn: a deployment that
+ * narrowed this run to three tools has not stopped supporting whatever the underlying agent
+ * supports.
+ */
+ getCapabilities() {
+ return this.whole.getCapabilities?.() ?? Promise.resolve({});
+ }
+
+ /**
+ * Carried by hand, because `AbstractAgent.clone` does not know this class exists.
+ *
+ * It builds a bare object on the prototype and copies a fixed list of base fields onto it, so
+ * every field declared here arrives `undefined`. The runtime clones an agent before every run
+ * (`agents[agentId].clone()`), which means the omission is not a corner case: without this, the
+ * first message anybody sends fails on a `build` that is not a function.
+ */
+ clone(): RunSelectedAgent {
+ const cloned = super.clone() as RunSelectedAgent;
+ cloned.whole = this.whole;
+ cloned.build = this.build;
+ // Deliberately not the inner agent. A clone is a new run, and inheriting the last run's agent
+ // would point `abortRun` at something already finished.
+ cloned.inner = undefined;
+ return cloned;
+ }
+
+ abortRun(): void {
+ this.inner?.abortRun();
+ super.abortRun();
+ }
+}
+
class UnavailableAgent extends AbstractAgent {
private readonly reason: string;
@@ -489,6 +710,7 @@ export async function resolveRuntimeAgents(
signRun?: SignRun,
computerGuidance?: string,
loadVendors?: () => Promise,
+ selection?: ToolSelection,
): Promise> {
const registered = await loadAgents();
if (registered.length === 0) {
@@ -509,6 +731,7 @@ export async function resolveRuntimeAgents(
signRun,
computerGuidance,
loadVendors,
+ selection,
);
}
@@ -558,6 +781,13 @@ export function createRequestAgents(
computerGuidance?: string,
/** Which vendors this deployment connects to, held by a Bot or not. Absent means none. */
loadVendors?: () => Promise,
+ /**
+ * How a run's tools are narrowed, resolved for whoever is asking.
+ *
+ * Per actor like the tools themselves, because the skills a Bot holds are read through the same
+ * grants, and because the discovery row has to name the person the run belongs to.
+ */
+ selectionForActor?: (actorId: string) => ToolSelection,
) {
return async ({ request }: { request: Request }) => {
const actor = await identifyActor(request);
@@ -570,6 +800,7 @@ export function createRequestAgents(
signRunForActor?.(actor.id),
computerGuidance,
loadVendors,
+ selectionForActor?.(actor.id),
);
};
}
@@ -598,6 +829,7 @@ export function mountCopilotRuntime(
signRunForActor?: (actorId: string) => SignRun,
basePath = "/api/copilotkit",
loadVendors?: () => Promise,
+ selectionForActor?: (actorId: string) => ToolSelection,
) {
const { intelligence } = config.runtime;
@@ -637,6 +869,7 @@ export function mountCopilotRuntime(
*/
config.computer ? COMPUTER_GUIDANCE : undefined,
loadVendors,
+ selectionForActor,
) as never,
});
diff --git a/server/src/index.ts b/server/src/index.ts
index e848e385..93bf2641 100644
--- a/server/src/index.ts
+++ b/server/src/index.ts
@@ -45,7 +45,7 @@ import {
import { createDatabase } from "./db/client";
import { createPeopleStore } from "./people/store";
import { createPluginStore } from "./plugins/store";
-import { grantedTools } from "./plugins/tools";
+import { grantedSkills, grantedTools } from "./plugins/tools";
import { createIntentRouter } from "./routing/classify";
import { createModelCompleter } from "./routing/model";
import {
@@ -374,6 +374,24 @@ const intentRouter = createIntentRouter({
}),
});
+/**
+ * Pass one of tool selection: which skills a message needs, on the deployment's own model.
+ *
+ * Built once rather than per request, because it holds nothing about a person: the key is resolved
+ * on every call, so a credential rotated a moment ago is used by the next run.
+ */
+const chooseSkills = createModelCompleter({
+ model: tenantPackage.model,
+ resolveApiKey: () =>
+ resolveModelApiKey({
+ encryptionKey: config.keyEncryptionKey,
+ reader: credentialStore,
+ provider: tenantPackage.model.provider,
+ keyId: tenantPackage.model.credentialSecretRef,
+ environment: process.env,
+ }),
+});
+
const app = createApp(
config,
auth,
@@ -435,6 +453,42 @@ const app = createApp(
return [];
}
},
+ /*
+ * How a run's tools are narrowed to the ones it is about.
+ *
+ * A model picks the right tool reliably out of about ten, and a deployment of this template
+ * clears that as soon as it connects a second vendor. Past it the wrong tool gets called, or
+ * none does and the answer comes from memory, and neither says so. So a Bot holding more than a
+ * handful is offered the tools of the skills that match the message rather than everything at
+ * once. See `plugins/selection.ts`.
+ *
+ * This narrows the offer and nothing else. What a Bot may call is the grant, checked in
+ * `callTool` with the policy and the audit row exactly as before, so every path through here can
+ * be wrong without a Bot gaining anything. That is also why every failure below is silent and
+ * lands on the whole catalogue: the narrowing is worth an accuracy point, never a capability.
+ */
+ (actorId) => ({
+ loadSkills: (botId) => grantedSkills({ store: pluginStore, botId }),
+ // The deployment's own model and key, the same pair the intent router uses, so selection is
+ // never a second thing to configure. It throws on a missing key, which reads as "could not
+ // choose" and leaves the whole catalogue offered.
+ choose: chooseSkills,
+ record: async (botId, selection) => {
+ await recordAuditEvent(bootAuditStore, {
+ eventType: "mcp.tools_discovered",
+ targetType: "bot",
+ targetId: botId,
+ actorUserId: actorId,
+ payload: {
+ bot: botId,
+ reason: selection.reason,
+ granted: selection.granted,
+ offered: selection.offered.length,
+ skills: selection.skills,
+ },
+ });
+ },
+ }),
),
// The only path to an acting call.
computerGateway,
diff --git a/server/src/plugins/selection.ts b/server/src/plugins/selection.ts
new file mode 100644
index 00000000..042bd4da
--- /dev/null
+++ b/server/src/plugins/selection.ts
@@ -0,0 +1,282 @@
+/**
+ * Choosing which of a Bot's tools to put in front of the model, one run at a time.
+ *
+ * WHY THIS EXISTS. A model picks the right tool reliably out of about ten. Past roughly fifteen the
+ * choice starts to go wrong, and it goes wrong quietly: the model calls a plausible neighbour, or
+ * calls nothing and answers from memory. A realistic deployment of this template clears fifteen on
+ * the first afternoon, because Drive and Slack and Jira and the browser each bring several. So the
+ * catalogue has to be narrowed before the model sees it, and the unit that does the narrowing is the
+ * skill: a skill says what it is for in one line, and it says which tools it needs.
+ *
+ * THE NARROWING IS NOT A BOUNDARY, AND MUST NEVER BE MISTAKEN FOR ONE. What a Bot may call is the
+ * grant, checked in `callTool` along with the policy and the audit row. This decides only what is
+ * offered out of what was already granted. Everything here can be wrong, or skipped entirely, and no
+ * Bot gains a single capability it did not already hold. That is why the failure direction below is
+ * "offer everything" rather than "offer nothing": narrowing is an accuracy device, and failing it
+ * closed would take away tools an administrator granted because a model call timed out.
+ *
+ * WHY THE MODEL CHOOSES AND NOT A RETRIEVER. A retrieval prefilter fails categorically. If the tool
+ * the run needed is not in the retrieved set, no amount of model capability gets it back, and the
+ * published result is that a prefilter at 99% recall can land at or below no prefilter at all for
+ * exactly that reason. A model that picks the wrong skill is wrong in a way the next turn can fix.
+ * A prefilter that drops the tool is wrong in a way nothing can. So the model chooses, retrieval (if
+ * a deployment ever needs it) narrows into that choice rather than replacing it, and every uncertain
+ * case here resolves towards offering more rather than less.
+ */
+
+/** One granted skill, as much of it as choosing needs. */
+export type SelectableSkill = {
+ slug: string;
+ title: string;
+ /** The one line the model reads. This is the index; see K3. */
+ summary: string;
+ /** What the skill says it needs, as `/` refs. A declaration, not a grant. */
+ tools: readonly string[];
+};
+
+/** A granted tool, as much of it as narrowing needs. */
+export type SelectableTool = {
+ /** `/`, the key a grant and a declaration are both written against. */
+ ref: string;
+};
+
+/**
+ * Why a run ended up offered what it was offered.
+ *
+ * Recorded rather than inferred, because every one of these looks identical from outside: the model
+ * was handed some tools. Which of them happened decides whether a wrong answer is a selection bug, a
+ * deployment that never declared anything, or a model call that failed. Without the reason, all
+ * three read as "the Bot did not use its tools".
+ */
+export type SelectionReason =
+ /** Few enough tools that a model chooses well among them unaided. Nothing was narrowed. */
+ | "under-floor"
+ /** No granted skill declares any granted tool, so there is no unit to select over. */
+ | "nothing-declared"
+ /** Pass one could not answer: no key, a timeout, a malformed reply. Everything stays offered. */
+ | "unavailable"
+ /** Pass one answered and named no skill. Everything stays offered; see the note below. */
+ | "nothing-chosen"
+ /** Pass one named skills, and the offer is their tools plus everything no skill claims. */
+ | "selected";
+
+export type Selection = {
+ /** What to hand the model. Always a subset of what was granted, and never a superset. */
+ offered: Tool[];
+ /** The slugs pass one chose. Empty for every reason other than `selected`. */
+ skills: string[];
+ reason: SelectionReason;
+ /** How many were granted, so a reader can see the narrowing without recomputing it. */
+ granted: number;
+};
+
+/**
+ * Below this, the catalogue is already inside the range a model chooses well from, so pass one is a
+ * model call that buys nothing and costs a round trip on every single run.
+ *
+ * Twelve because the reported knee is ten to fifteen and the cost of being slightly under it is
+ * nothing, while the cost of being over it is a wrong tool call nobody sees. This is a template's
+ * default, not a law: a deployment that measures its own knee somewhere else should move it.
+ */
+export const SELECTION_FLOOR = 12;
+
+/**
+ * What pass one is asked, given the message and the skills the Bot holds.
+ *
+ * Deliberately biased towards choosing. The two mistakes are not symmetrical: an extra skill costs a
+ * few tool definitions in the context, and a missing one costs the answer, because the tool it would
+ * have loaded is not there to call. The prompt says so in as many words rather than leaving the
+ * model to guess the trade, and the caller treats an empty answer as "offer everything" for the same
+ * reason.
+ */
+export function selectionPrompt(
+ text: string,
+ skills: readonly SelectableSkill[],
+): string {
+ const catalogue = skills
+ .map((skill) => `- ${skill.slug}: ${skill.title}. ${skill.summary}`)
+ .join("\n");
+ return [
+ "You choose which capabilities to load for the message below. You are not answering it.",
+ "",
+ "Capabilities available:",
+ catalogue,
+ "",
+ "Message:",
+ text,
+ "",
+ 'Reply with only JSON: {"skills": ["", ...]}.',
+ "Choose every capability that might be needed, including ones you are only somewhat sure about.",
+ "Choosing one that turns out to be unnecessary costs almost nothing. Failing to choose one that",
+ "was needed means the work cannot be done at all, because its tools will not be loaded. When in",
+ "doubt, include it. Use an empty list only when the message plainly needs none of them.",
+ ].join("\n");
+}
+
+/**
+ * Read pass one's answer into slugs, or `null` when it did not answer usefully.
+ *
+ * `null` and `[]` mean different things and the caller treats them differently: `null` is "the
+ * selector did not work", `[]` is "the selector says none apply". Both currently end at the same
+ * place, offering everything, but they are different facts and the audit row records which.
+ *
+ * Anything the model names that is not a granted skill is dropped rather than treated as an error.
+ * A model inventing a slug should cost that slug, not the whole selection.
+ */
+export function readChosenSkills(
+ answer: string,
+ skills: readonly SelectableSkill[],
+): string[] | null {
+ let parsed: unknown;
+ try {
+ parsed = JSON.parse(answer);
+ } catch {
+ return null;
+ }
+ if (typeof parsed !== "object" || parsed === null) return null;
+ const chosen = (parsed as { skills?: unknown }).skills;
+ if (!Array.isArray(chosen)) return null;
+ const known = new Set(skills.map((skill) => skill.slug));
+ return [
+ ...new Set(
+ chosen.filter(
+ (slug): slug is string => typeof slug === "string" && known.has(slug),
+ ),
+ ),
+ ];
+}
+
+/**
+ * The tools a set of chosen skills asks for, intersected with what the Bot actually holds.
+ *
+ * The intersection is the whole safety property. A skill may name any tool: anybody signed in may
+ * write one, and `skill_tools` deliberately has no foreign key and grants nothing. If a declaration
+ * could widen the offer, writing a skill would be a way to hand yourself a tool, and the one surface
+ * here that is not an administrator's would become the way around every surface that is.
+ */
+function declaredBy(
+ skills: readonly SelectableSkill[],
+ granted: ReadonlySet,
+): Set {
+ const refs = new Set();
+ for (const skill of skills) {
+ for (const ref of skill.tools) if (granted.has(ref)) refs.add(ref);
+ }
+ return refs;
+}
+
+/**
+ * Narrow one Bot's granted tools for one run.
+ *
+ * `choose` is pass one, injected rather than called here so this stays a plain function a test can
+ * drive without a network. It may throw or return `null`; both mean "could not say", and both leave
+ * every granted tool offered.
+ */
+export async function selectTools(input: {
+ tools: readonly Tool[];
+ skills: readonly SelectableSkill[];
+ /** The message this run is about. Empty is treated as "cannot say", not as "needs nothing". */
+ text: string;
+ choose: (
+ prompt: string,
+ ) => Promise | (string | null) | Promise;
+ /** Overridable so a deployment that measured its own knee is not stuck with ours. */
+ floor?: number;
+}): Promise> {
+ const { tools, skills, text } = input;
+ const floor = input.floor ?? SELECTION_FLOOR;
+ const everything = (reason: SelectionReason): Selection => ({
+ offered: [...tools],
+ skills: [],
+ reason,
+ granted: tools.length,
+ });
+
+ if (tools.length <= floor) return everything("under-floor");
+
+ const grantedRefs = new Set(tools.map((tool) => tool.ref));
+ const declared = declaredBy(skills, grantedRefs);
+ // Nothing to select over. A Bot with grants and no skills is every deployment on day one, and it
+ // must behave exactly as it did before this existed.
+ if (declared.size === 0) return everything("nothing-declared");
+ if (text.trim() === "") return everything("unavailable");
+
+ let chosen: string[] | null = null;
+ try {
+ const answer = await input.choose(selectionPrompt(text, skills));
+ chosen =
+ typeof answer === "string" ? readChosenSkills(answer, skills) : null;
+ } catch {
+ // A selector that failed is not an error a person should ever see. It costs this run the
+ // narrowing and nothing else, which is the behaviour that shipped before it existed.
+ chosen = null;
+ }
+ if (chosen === null) return everything("unavailable");
+ /*
+ * The model says none apply, and everything stays offered anyway.
+ *
+ * Reading this as "offer only the tools no skill claims" would be the categorical failure the
+ * header warns about: one bad judgement in pass one, and the tool the run needed is not merely
+ * ranked low, it is absent. Offering everything here is the behaviour that shipped before
+ * selection existed, so the worst case of a confused selector is exactly the old accuracy rather
+ * than a Bot that has lost its hands.
+ */
+ if (chosen.length === 0) return everything("nothing-chosen");
+
+ const wanted = declaredBy(
+ skills.filter((skill) => chosen.includes(skill.slug)),
+ grantedRefs,
+ );
+ return {
+ /*
+ * The chosen skills' tools, plus every granted tool no skill claims at all.
+ *
+ * Undeclared tools ride along on purpose. A declaration is opt-in, so an administrator can grant
+ * a tool that no skill has been written for yet, and dropping it would silently remove a
+ * capability somebody deliberately handed over. The offer therefore shrinks as skills come to
+ * cover the catalogue, and a deployment that has declared nothing is never punished for it.
+ */
+ offered: tools.filter(
+ (tool) => !declared.has(tool.ref) || wanted.has(tool.ref),
+ ),
+ skills: chosen,
+ reason: "selected",
+ granted: tools.length,
+ };
+}
+
+/**
+ * The message pass one reads: the last thing the person said.
+ *
+ * The last user message rather than the whole thread, because what to load is a question about the
+ * turn being taken. Feeding the transcript in would make an early mention of Drive keep Drive tools
+ * loaded for the rest of the conversation, which is the opposite of narrowing.
+ */
+export function latestUserText(
+ messages: readonly { role?: string; content?: unknown }[],
+): string {
+ for (let index = messages.length - 1; index >= 0; index -= 1) {
+ const message = messages[index];
+ if (message?.role !== "user") continue;
+ if (typeof message.content === "string") return message.content;
+ // AG-UI allows structured content. Text parts are the only part a selector can read.
+ if (Array.isArray(message.content)) {
+ const text = message.content
+ .map((part) =>
+ typeof part === "object" &&
+ part !== null &&
+ typeof (part as { text?: unknown }).text === "string"
+ ? ((part as { text: string }).text as string)
+ : "",
+ )
+ // Dropped before joining, so an image between two sentences does not leave a double space
+ // in the middle of the one thing the selector reads.
+ .filter((part) => part !== "")
+ .join(" ")
+ .trim();
+ if (text !== "") return text;
+ }
+ return "";
+ }
+ return "";
+}
diff --git a/server/src/plugins/tools.ts b/server/src/plugins/tools.ts
index 297cf87c..d4bb9a2f 100644
--- a/server/src/plugins/tools.ts
+++ b/server/src/plugins/tools.ts
@@ -1,4 +1,5 @@
import { z } from "zod";
+import type { SelectableSkill } from "./selection";
import { PluginRefusedError, type PluginStore } from "./store";
/**
@@ -31,6 +32,15 @@ export type GrantedTool = {
description: string;
parameters: z.ZodType;
execute: (args: unknown) => Promise;
+ /**
+ * `/`, carried alongside the name the model is offered.
+ *
+ * The two spellings exist because a model tool name may not contain a slash, and selection has to
+ * compare a tool against what a skill declared, which is written in the ref spelling because that
+ * is how a grant is written. Carried rather than derived at the comparison, so there is one place
+ * the two forms are converted (`toolNameFor`) and no second parser to drift from it.
+ */
+ ref: string;
};
/**
@@ -154,6 +164,7 @@ export async function grantedTools(options: {
return granted.tools.map((tool) => ({
name: tool.toolName,
+ ref: tool.ref,
description: tool.description,
parameters: parametersFor(tool.inputSchema),
execute: async (args: unknown) => {
@@ -198,3 +209,24 @@ export async function grantedTools(options: {
},
}));
}
+
+/**
+ * The skills one Bot holds, as much of each as choosing between them needs.
+ *
+ * Read here rather than folded into `grantedTools` because the two answer different questions and
+ * are wanted at different moments: the tools are what a Bot may call, asked once per request, and
+ * the skills are the index a run is narrowed against. Both come from `listForAgent`, and both are
+ * read fresh for the same reason: a grant added a minute ago has to count on the next run.
+ */
+export async function grantedSkills(options: {
+ store: PluginStore;
+ botId: string;
+}): Promise {
+ const granted = await options.store.listForAgent(options.botId);
+ return granted.skills.map((skill) => ({
+ slug: skill.slug,
+ title: skill.title,
+ summary: skill.summary,
+ tools: skill.tools,
+ }));
+}
diff --git a/server/src/routing/model.ts b/server/src/routing/model.ts
index 1bbb4e41..67794280 100644
--- a/server/src/routing/model.ts
+++ b/server/src/routing/model.ts
@@ -15,9 +15,7 @@ export function createModelCompleter(deps: {
return async (prompt: string) => {
const key = await deps.resolveApiKey();
if (!key) throw new Error("no model key");
- const base =
- process.env.OPENAI_BASE_URL?.trim() || "https://api.openai.com";
- const response = await fetch(`${base}/v1/chat/completions`, {
+ const response = await fetch(chatCompletionsUrl(process.env), {
method: "POST",
headers: {
"content-type": "application/json",
@@ -55,3 +53,32 @@ export function createModelCompleter(deps: {
return content;
};
}
+
+/**
+ * Where this deployment's `/chat/completions` actually is.
+ *
+ * `/v1` USED TO BE APPENDED UNCONDITIONALLY, and that was wrong for every deployment that set the
+ * variable. `.env.example` and `docs/configuration.md` both document it with the version in it
+ * (`https://gateway.internal/v1`), because that is the shape the AI SDK wants: it takes `baseURL`
+ * verbatim and asks for `/chat/completions` under it, which is how the built-in Bots reach the same
+ * endpoint. Appending here produced `/v1/v1/chat/completions`, so on a gateway — the entire reason
+ * the variable exists — every call from this function 404'd.
+ *
+ * That failure was invisible, which is the worst part. The router treats a throw as "not sure" and
+ * lands on the default coworker, so a deployment behind a gateway silently stopped routing and
+ * nothing anywhere said why. Tool selection reads through the same function and would have failed
+ * the same way, offering the whole catalogue on the deployments most likely to have a big one.
+ *
+ * So the version segment is added only when the configured URL does not already end in one, and the
+ * unset case keeps the public API's own `https://api.openai.com/v1`.
+ */
+export function chatCompletionsUrl(
+ environment: Record,
+): string {
+ const base = (environment.OPENAI_BASE_URL?.trim() || "https://api.openai.com")
+ // A trailing slash is the difference between `/v1` and `/v1/`, and no more than that.
+ .replace(/\/+$/, "");
+ return /\/v\d+$/.test(base)
+ ? `${base}/chat/completions`
+ : `${base}/v1/chat/completions`;
+}
diff --git a/server/tests/plugin-selection.test.ts b/server/tests/plugin-selection.test.ts
new file mode 100644
index 00000000..9ae8f77e
--- /dev/null
+++ b/server/tests/plugin-selection.test.ts
@@ -0,0 +1,333 @@
+import { describe, expect, test } from "bun:test";
+import {
+ latestUserText,
+ readChosenSkills,
+ SELECTION_FLOOR,
+ selectionPrompt,
+ selectTools,
+} from "../src/plugins/selection";
+
+/**
+ * Narrowing a Bot's tools to the ones a run is about.
+ *
+ * Two properties are asserted harder than the rest, because they are the two that would make this
+ * dangerous rather than merely wrong. The first is that a skill cannot widen anything: a declaration
+ * naming a tool the Bot was never granted must produce nothing, or writing a skill — which anybody
+ * signed in may do — becomes a way to grant yourself a tool. The second is the failure direction:
+ * every way this can go wrong has to end with the whole catalogue offered, because a selector that
+ * fails closed takes away tools an administrator granted, and does it silently.
+ */
+
+const tool = (ref: string) => ({ ref });
+
+/** More than `SELECTION_FLOOR`, so selection actually runs. Named for what it is doing. */
+const manyTools = [
+ ...Array.from({ length: 8 }, (_, index) => tool(`drive/tool_${index}`)),
+ ...Array.from({ length: 8 }, (_, index) => tool(`slack/tool_${index}`)),
+];
+
+const skills = [
+ {
+ slug: "drive-audit",
+ title: "Drive audit",
+ summary: "Read documents out of Google Drive.",
+ tools: ["drive/tool_0", "drive/tool_1"],
+ },
+ {
+ slug: "slack-digest",
+ title: "Slack digest",
+ summary: "Summarise Slack channels.",
+ tools: ["slack/tool_0"],
+ },
+];
+
+const answering = (skillSlugs: string[]) => async () =>
+ JSON.stringify({ skills: skillSlugs });
+
+describe("what gets offered", () => {
+ test("a small catalogue is offered whole, and pass one is never called", async () => {
+ let asked = 0;
+ const tools = Array.from({ length: SELECTION_FLOOR }, (_, index) =>
+ tool(`drive/tool_${index}`),
+ );
+ const selection = await selectTools({
+ tools,
+ skills,
+ text: "read my drive",
+ choose: async () => {
+ asked += 1;
+ return JSON.stringify({ skills: [] });
+ },
+ });
+ expect(selection.reason).toBe("under-floor");
+ expect(selection.offered).toHaveLength(SELECTION_FLOOR);
+ // The point of the floor is the round trip it saves, so the assertion is that it was saved.
+ expect(asked).toBe(0);
+ });
+
+ test("chosen skills bring their tools, and the rest of their servers stay behind", async () => {
+ const selection = await selectTools({
+ tools: manyTools,
+ skills,
+ text: "what is in the quarterly report in my drive",
+ choose: answering(["drive-audit"]),
+ });
+ expect(selection.reason).toBe("selected");
+ expect(selection.skills).toEqual(["drive-audit"]);
+ expect(selection.offered.map((entry) => entry.ref)).toContain(
+ "drive/tool_0",
+ );
+ expect(selection.offered.map((entry) => entry.ref)).toContain(
+ "drive/tool_1",
+ );
+ // Declared by the other skill, and that skill was not chosen.
+ expect(selection.offered.map((entry) => entry.ref)).not.toContain(
+ "slack/tool_0",
+ );
+ expect(selection.granted).toBe(manyTools.length);
+ });
+
+ test("a tool no skill declares is always offered", async () => {
+ const selection = await selectTools({
+ tools: manyTools,
+ skills,
+ text: "anything",
+ choose: answering(["drive-audit"]),
+ });
+ const offered = selection.offered.map((entry) => entry.ref);
+ /*
+ * `slack/tool_1` onwards are granted and claimed by nobody. Dropping them would silently remove
+ * a capability an administrator handed over, on the strength of a skill nobody has written yet.
+ */
+ expect(offered).toContain("slack/tool_1");
+ expect(offered).toContain("drive/tool_7");
+ });
+
+ test("choosing several skills unions their tools", async () => {
+ const selection = await selectTools({
+ tools: manyTools,
+ skills,
+ text: "compare the drive doc against the slack thread",
+ choose: answering(["drive-audit", "slack-digest"]),
+ });
+ const offered = selection.offered.map((entry) => entry.ref);
+ expect(offered).toContain("drive/tool_0");
+ expect(offered).toContain("slack/tool_0");
+ });
+});
+
+describe("a declaration is not a grant", () => {
+ test("a skill naming a tool the Bot does not hold offers nothing extra", async () => {
+ const selection = await selectTools({
+ tools: manyTools,
+ skills: [
+ {
+ slug: "overreach",
+ title: "Overreach",
+ summary: "Names a tool nobody granted.",
+ // Both are absent from `manyTools`. One is a server the Bot has, one is a server it
+ // does not; neither may appear, and neither may cause the others to be dropped.
+ tools: ["drive/delete_everything", "vault/read_secret"],
+ },
+ ...skills,
+ ],
+ text: "delete everything",
+ choose: answering(["overreach"]),
+ });
+ const offered = selection.offered.map((entry) => entry.ref);
+ expect(offered).not.toContain("drive/delete_everything");
+ expect(offered).not.toContain("vault/read_secret");
+ // The offer is still a subset of the grants, which is the invariant that matters.
+ for (const entry of offered) {
+ expect(manyTools.map((granted) => granted.ref)).toContain(entry);
+ }
+ });
+
+ test("the offer is never larger than the grant, whatever is chosen", async () => {
+ const selection = await selectTools({
+ tools: manyTools,
+ skills,
+ text: "everything at once",
+ choose: answering(["drive-audit", "slack-digest"]),
+ });
+ expect(selection.offered.length).toBeLessThanOrEqual(manyTools.length);
+ });
+});
+
+describe("every failure offers everything", () => {
+ test("a selector that throws", async () => {
+ const selection = await selectTools({
+ tools: manyTools,
+ skills,
+ text: "read my drive",
+ choose: async () => {
+ throw new Error("no model key");
+ },
+ });
+ expect(selection.reason).toBe("unavailable");
+ expect(selection.offered).toHaveLength(manyTools.length);
+ });
+
+ test("a selector that answers with something that is not JSON", async () => {
+ const selection = await selectTools({
+ tools: manyTools,
+ skills,
+ text: "read my drive",
+ choose: async () => "I think you want the Drive one",
+ });
+ expect(selection.reason).toBe("unavailable");
+ expect(selection.offered).toHaveLength(manyTools.length);
+ });
+
+ test("a selector that answers with JSON of the wrong shape", async () => {
+ const selection = await selectTools({
+ tools: manyTools,
+ skills,
+ text: "read my drive",
+ choose: async () => JSON.stringify({ chosen: "drive-audit" }),
+ });
+ expect(selection.reason).toBe("unavailable");
+ expect(selection.offered).toHaveLength(manyTools.length);
+ });
+
+ test("a selector that names no skill at all", async () => {
+ const selection = await selectTools({
+ tools: manyTools,
+ skills,
+ text: "hello",
+ choose: answering([]),
+ });
+ /*
+ * Distinguished from `unavailable` in the trail, and identical in effect. Reading an empty
+ * answer as "offer only the undeclared tools" would mean one bad judgement in pass one makes the
+ * needed tool absent rather than merely unlikely, which is the categorical failure this design
+ * exists to avoid.
+ */
+ expect(selection.reason).toBe("nothing-chosen");
+ expect(selection.offered).toHaveLength(manyTools.length);
+ });
+
+ test("no skill declares anything", async () => {
+ const selection = await selectTools({
+ tools: manyTools,
+ skills: [
+ {
+ slug: "prose",
+ title: "Prose",
+ summary: "Just instructions",
+ tools: [],
+ },
+ ],
+ text: "read my drive",
+ choose: async () => {
+ throw new Error("should not be asked");
+ },
+ });
+ expect(selection.reason).toBe("nothing-declared");
+ expect(selection.offered).toHaveLength(manyTools.length);
+ });
+
+ test("an empty message", async () => {
+ const selection = await selectTools({
+ tools: manyTools,
+ skills,
+ text: " ",
+ choose: async () => {
+ throw new Error("should not be asked");
+ },
+ });
+ expect(selection.reason).toBe("unavailable");
+ expect(selection.offered).toHaveLength(manyTools.length);
+ });
+});
+
+describe("reading pass one's answer", () => {
+ test("a slug that is not a granted skill is dropped, not fatal", () => {
+ expect(
+ readChosenSkills(
+ JSON.stringify({ skills: ["drive-audit", "invented"] }),
+ skills,
+ ),
+ ).toEqual(["drive-audit"]);
+ });
+
+ test("duplicates collapse", () => {
+ expect(
+ readChosenSkills(
+ JSON.stringify({ skills: ["drive-audit", "drive-audit"] }),
+ skills,
+ ),
+ ).toEqual(["drive-audit"]);
+ });
+
+ test("non-strings inside the list are dropped", () => {
+ expect(
+ readChosenSkills(
+ JSON.stringify({ skills: [1, null, "slack-digest"] }),
+ skills,
+ ),
+ ).toEqual(["slack-digest"]);
+ });
+
+ test("null for anything that is not an object with a list", () => {
+ expect(readChosenSkills("[]", skills)).toBeNull();
+ expect(readChosenSkills("null", skills)).toBeNull();
+ expect(readChosenSkills("{}", skills)).toBeNull();
+ expect(readChosenSkills("not json", skills)).toBeNull();
+ });
+
+ test("an answer naming only unknown slugs is an empty choice, not a failure", () => {
+ // The model answered; it just named nothing real. That is "none apply", and the caller offers
+ // everything either way, but the two are different facts and the row says which.
+ expect(
+ readChosenSkills(JSON.stringify({ skills: ["nope"] }), skills),
+ ).toEqual([]);
+ });
+});
+
+describe("the prompt", () => {
+ test("carries every skill and the message, and says which way to err", () => {
+ const prompt = selectionPrompt("find the quarterly report", skills);
+ expect(prompt).toContain("drive-audit");
+ expect(prompt).toContain("Read documents out of Google Drive.");
+ expect(prompt).toContain("slack-digest");
+ expect(prompt).toContain("find the quarterly report");
+ // The asymmetry is the whole reason pass one is safe. If the prompt stops saying it, a model
+ // will start being parsimonious and the misses become invisible.
+ expect(prompt).toContain("When in");
+ expect(prompt).toContain("doubt, include it.");
+ });
+});
+
+describe("which message pass one reads", () => {
+ test("the last user message, not the last message", () => {
+ expect(
+ latestUserText([
+ { role: "user", content: "first" },
+ { role: "assistant", content: "an answer" },
+ { role: "user", content: "second" },
+ { role: "assistant", content: "another answer" },
+ ]),
+ ).toBe("second");
+ });
+
+ test("structured content is flattened to its text parts", () => {
+ expect(
+ latestUserText([
+ {
+ role: "user",
+ content: [
+ { type: "text", text: "look at" },
+ { type: "image", url: "http://example.test/x.png" },
+ { type: "text", text: "this" },
+ ],
+ },
+ ]),
+ ).toBe("look at this");
+ });
+
+ test("no user message at all reads as nothing to select against", () => {
+ expect(latestUserText([{ role: "system", content: "a role" }])).toBe("");
+ expect(latestUserText([])).toBe("");
+ });
+});
diff --git a/server/tests/routing-model-url.test.ts b/server/tests/routing-model-url.test.ts
new file mode 100644
index 00000000..31e2af59
--- /dev/null
+++ b/server/tests/routing-model-url.test.ts
@@ -0,0 +1,59 @@
+import { describe, expect, test } from "bun:test";
+import { chatCompletionsUrl } from "../src/routing/model";
+
+/**
+ * Where the deployment's own model calls go.
+ *
+ * This existed as `${base}/v1/chat/completions` and the documented value of `OPENAI_BASE_URL`
+ * already ends in `/v1`, so every deployment behind a gateway got `/v1/v1/chat/completions` and a
+ * 404. Both callers treat a throw as "not sure": the intent router silently routed everything to
+ * the default coworker, and tool selection would have silently offered every tool. Neither said
+ * anything, which is why the case is pinned here rather than left to a comment.
+ */
+describe("chatCompletionsUrl", () => {
+ test("unset falls back to the public API, with its version", () => {
+ expect(chatCompletionsUrl({})).toBe(
+ "https://api.openai.com/v1/chat/completions",
+ );
+ });
+
+ test("a gateway documented with /v1 is not given a second one", () => {
+ expect(
+ chatCompletionsUrl({ OPENAI_BASE_URL: "https://gateway.internal/v1" }),
+ ).toBe("https://gateway.internal/v1/chat/completions");
+ });
+
+ test("a trailing slash is not a different URL", () => {
+ expect(
+ chatCompletionsUrl({ OPENAI_BASE_URL: "https://gateway.internal/v1/" }),
+ ).toBe("https://gateway.internal/v1/chat/completions");
+ });
+
+ test("a host with no version gets one, which is what a bare origin means", () => {
+ expect(
+ chatCompletionsUrl({ OPENAI_BASE_URL: "http://localhost:4010" }),
+ ).toBe("http://localhost:4010/v1/chat/completions");
+ });
+
+ test("a version other than 1 is still a version", () => {
+ expect(chatCompletionsUrl({ OPENAI_BASE_URL: "https://x.test/v2" })).toBe(
+ "https://x.test/v2/chat/completions",
+ );
+ });
+
+ test("whitespace and empty are the same as unset", () => {
+ expect(chatCompletionsUrl({ OPENAI_BASE_URL: " " })).toBe(
+ "https://api.openai.com/v1/chat/completions",
+ );
+ expect(chatCompletionsUrl({ OPENAI_BASE_URL: "" })).toBe(
+ "https://api.openai.com/v1/chat/completions",
+ );
+ });
+
+ test("a path that merely contains v1 is not a version segment", () => {
+ // `/v1beta` is Google's, and it is not the segment this is looking for.
+ expect(
+ chatCompletionsUrl({ OPENAI_BASE_URL: "https://x.test/v1beta" }),
+ ).toBe("https://x.test/v1beta/v1/chat/completions");
+ });
+});
diff --git a/server/tests/tool-selection.integration.test.ts b/server/tests/tool-selection.integration.test.ts
new file mode 100644
index 00000000..474b303d
--- /dev/null
+++ b/server/tests/tool-selection.integration.test.ts
@@ -0,0 +1,554 @@
+import {
+ afterAll,
+ beforeAll,
+ beforeEach,
+ describe,
+ expect,
+ test,
+} from "bun:test";
+import { buildAGUITextResponse, LLMock } from "@copilotkit/aimock";
+import { AGUIMock } from "@copilotkit/aimock/agui";
+import { z } from "zod";
+import {
+ buildAgents,
+ type RegisteredAgent,
+ type RuntimeModel,
+} from "../src/copilot";
+import type { Selection } from "../src/plugins/selection";
+import type { GrantedTool } from "../src/plugins/tools";
+import { createModelCompleter } from "../src/routing/model";
+
+/**
+ * Tool selection, asserted on the bytes that reach the model rather than on the decision.
+ *
+ * The unit tests next door prove `selectTools` narrows correctly. They cannot prove the narrowing
+ * arrives: the tools are attached at agent construction, the runtime clones the agent before every
+ * run, and both of those sit between the decision and the request. So this drives the real
+ * `buildAgents`, through a real clone, against `@copilotkit/aimock` — ours, the org's deterministic
+ * backend — and reads the tool list out of the request the mock actually received. If the agent were
+ * built with the whole catalogue, or the clone lost the wrapper, or pass one never happened, the
+ * decision would still be right and every one of these would fail.
+ *
+ * Both paths are covered because they attach tools differently and would break separately: a
+ * built-in Bot carries them in its configuration, a remote one is sent them in the AG-UI run body.
+ */
+
+const model: RuntimeModel = { provider: "openai", defaultModel: "gpt-5.5" };
+
+/** Sixteen tools across two servers: over the floor, so selection has something to do. */
+const granted: GrantedTool[] = [
+ ...Array.from({ length: 8 }, (_, index) => grantedTool("drive", index)),
+ ...Array.from({ length: 8 }, (_, index) => grantedTool("slack", index)),
+];
+
+function grantedTool(server: string, index: number): GrantedTool {
+ const ref = `${server}/tool_${index}`;
+ return {
+ ref,
+ name: `mcp__${server}__tool_${index}`,
+ description: `${server} tool ${index}`,
+ parameters: z.object({ q: z.string() }),
+ execute: async () => "ok",
+ };
+}
+
+const skills = [
+ {
+ slug: "drive-audit",
+ title: "Drive audit",
+ summary: "Read documents out of Google Drive.",
+ tools: ["drive/tool_0", "drive/tool_1"],
+ },
+ {
+ slug: "slack-digest",
+ title: "Slack digest",
+ summary: "Summarise Slack channels.",
+ // Every Slack tool, so a Slack tool being offered can only mean this skill was chosen.
+ tools: Array.from({ length: 8 }, (_, index) => `slack/tool_${index}`),
+ },
+];
+
+const llm = new LLMock();
+const remote = new AGUIMock();
+let remoteUrl = "";
+/** Every AG-UI run the mock received, as the endpoint saw it. */
+let sentToRemote: {
+ tools: string[];
+ messages: { id?: string; role?: string; content?: unknown }[];
+ forwardedProps: Record;
+}[] = [];
+
+beforeAll(async () => {
+ const url = await llm.start();
+ process.env.OPENAI_BASE_URL = url;
+ process.env.OPENAI_API_KEY = "test-key";
+
+ remote.onPredicate(
+ (input) => {
+ sentToRemote.push({
+ tools: ((input.tools ?? []) as { name?: string }[])
+ .map((tool) => tool.name ?? "")
+ .filter(Boolean),
+ messages: (input.messages ?? []) as never,
+ forwardedProps: (input.forwardedProps ?? {}) as Record,
+ });
+ return true;
+ },
+ // Built rather than hand-written: the events carry the run and thread ids the protocol requires,
+ // and the client verifies them, so a hand-rolled sequence fails validation rather than the test.
+ buildAGUITextResponse("done") as never,
+ );
+ remoteUrl = await remote.start();
+});
+
+afterAll(async () => {
+ await llm.stop();
+ await remote.stop();
+});
+
+beforeEach(() => {
+ llm.clearRequests();
+ llm.clearFixtures();
+ sentToRemote = [];
+});
+
+/**
+ * Pass one answers with `chosen`, and the run itself answers with prose.
+ *
+ * Ordered: the selection prompt is matched first by its own opening line, and everything else falls
+ * through to the second fixture. Matching pass one on text the prompt actually contains is the point
+ * — if the prompt is ever rewritten without it, this stops matching and the tests fail loudly rather
+ * than quietly testing the un-narrowed path.
+ */
+function answerWith(chosen: string[]) {
+ llm.onMessage(/You choose which capabilities to load/, {
+ type: "text",
+ content: JSON.stringify({ skills: chosen }),
+ });
+ llm.onMessage(/.*/, { type: "text", content: "Here is what I found." });
+}
+
+const recorded: Selection[] = [];
+
+function selection(overrides: { floor?: number } = {}) {
+ return {
+ loadSkills: async () => skills,
+ choose: createModelCompleter({
+ model,
+ resolveApiKey: async () => "test-key",
+ }),
+ record: async (_botId: string, entry: Selection) => {
+ recorded.push(entry);
+ },
+ ...overrides,
+ };
+}
+
+const builtIn: RegisteredAgent = {
+ id: "analyst",
+ name: "Analyst",
+ type: "built_in",
+ systemPrompt: "You are an analyst.",
+};
+
+const remoteAgent = (): RegisteredAgent => ({
+ id: "risk",
+ name: "Risk",
+ type: "remote_ag_ui",
+ endpoint: `${remoteUrl}/`,
+ standingMessage: {
+ id: "standing-role:risk",
+ role: "system",
+ content: "You are Risk.",
+ },
+});
+
+/**
+ * Run one Bot the way the runtime does, including the clone.
+ *
+ * `agents[agentId].clone()` is what the runtime calls before every run, and `AbstractAgent.clone`
+ * copies a fixed list of its own fields onto a bare object — it knows nothing about a subclass. A
+ * wrapper that did not carry its own state across would fail here and nowhere else.
+ */
+async function ask(agent: { clone: () => unknown }, text: string) {
+ const running = (agent.clone as () => never)() as unknown as {
+ addMessage: (message: unknown) => void;
+ runAgent: () => Promise;
+ };
+ running.addMessage({ id: `m-${text.length}`, role: "user", content: text });
+ await running.runAgent();
+}
+
+/** The tool names in the last request the model actually received for a run (not for pass one). */
+function toolsOfferedToModel(): string[] {
+ const runs = llm
+ .getRequests()
+ .filter((entry) =>
+ Array.isArray((entry.body as { tools?: unknown })?.tools),
+ );
+ const last = runs.at(-1);
+ return (
+ (last?.body as { tools?: { function?: { name?: string } }[] })?.tools ?? []
+ )
+ .map((tool) => tool.function?.name ?? "")
+ .filter((name) => name.startsWith("mcp__"));
+}
+
+describe("a built-in Bot", () => {
+ test("is offered the chosen skill's tools and the tools no skill claims", async () => {
+ answerWith(["drive-audit"]);
+ const agents = await buildAgents(
+ [builtIn],
+ model,
+ "test-key",
+ undefined,
+ async () => granted,
+ undefined,
+ undefined,
+ undefined,
+ selection(),
+ );
+
+ await ask(
+ agents.analyst as never,
+ "what is in the quarterly report in Drive",
+ );
+
+ const offered = toolsOfferedToModel();
+ // Declared by the chosen skill.
+ expect(offered).toContain("mcp__drive__tool_0");
+ expect(offered).toContain("mcp__drive__tool_1");
+ // Granted, and claimed by no skill at all, so still offered.
+ expect(offered).toContain("mcp__drive__tool_7");
+ // Declared only by the skill that was not chosen. This is the narrowing.
+ expect(offered).not.toContain("mcp__slack__tool_0");
+ expect(offered).not.toContain("mcp__slack__tool_7");
+ expect(offered).toHaveLength(8);
+ });
+
+ test("pass one really happened, against the real endpoint", async () => {
+ answerWith(["drive-audit"]);
+ const agents = await buildAgents(
+ [builtIn],
+ model,
+ "test-key",
+ undefined,
+ async () => granted,
+ undefined,
+ undefined,
+ undefined,
+ selection(),
+ );
+ await ask(agents.analyst as never, "read the Drive doc");
+
+ const prompts = llm
+ .getRequests()
+ .flatMap((entry) =>
+ ((entry.body as { messages?: { content?: unknown }[] })?.messages ?? [])
+ .map((message) => message.content)
+ .filter((content): content is string => typeof content === "string"),
+ );
+ expect(
+ prompts.some((prompt) =>
+ prompt.includes("You choose which capabilities to load"),
+ ),
+ ).toBe(true);
+ });
+
+ test("both skills chosen offers both their tools", async () => {
+ answerWith(["drive-audit", "slack-digest"]);
+ const agents = await buildAgents(
+ [builtIn],
+ model,
+ "test-key",
+ undefined,
+ async () => granted,
+ undefined,
+ undefined,
+ undefined,
+ selection(),
+ );
+ await ask(
+ agents.analyst as never,
+ "compare the Drive doc with the Slack thread",
+ );
+
+ const offered = toolsOfferedToModel();
+ expect(offered).toContain("mcp__drive__tool_0");
+ expect(offered).toContain("mcp__slack__tool_0");
+ expect(offered).toHaveLength(granted.length);
+ });
+
+ test("a model that cannot answer costs the narrowing and not the tools", async () => {
+ // No fixture for the selection prompt: aimock has nothing to serve, so pass one fails the way a
+ // real outage does, and the run has to carry on with everything.
+ llm.onMessage(/.*/, { type: "text", content: "Here is what I found." });
+ const agents = await buildAgents(
+ [builtIn],
+ model,
+ "test-key",
+ undefined,
+ async () => granted,
+ undefined,
+ undefined,
+ undefined,
+ {
+ loadSkills: async () => skills,
+ choose: async () => {
+ throw new Error("model unreachable");
+ },
+ },
+ );
+ await ask(agents.analyst as never, "read the Drive doc");
+
+ expect(toolsOfferedToModel()).toHaveLength(granted.length);
+ });
+
+ test("the guidance names only what the run was offered", async () => {
+ answerWith(["drive-audit"]);
+ const agents = await buildAgents(
+ [builtIn],
+ model,
+ "test-key",
+ undefined,
+ async () => granted,
+ undefined,
+ undefined,
+ undefined,
+ selection(),
+ );
+ await ask(agents.analyst as never, "read the Drive doc");
+
+ const runs = llm
+ .getRequests()
+ .filter((entry) =>
+ Array.isArray((entry.body as { tools?: unknown })?.tools),
+ );
+ const system = (
+ (
+ runs.at(-1)?.body as {
+ messages?: { role?: string; content?: unknown }[];
+ }
+ )?.messages ?? []
+ )
+ .filter((message) => message.role === "system")
+ .map((message) => String(message.content))
+ .join("\n");
+ /*
+ * A Bot told it holds Slack tools it was not offered will promise Slack and then be unable to
+ * do it, which reads to the person as the Bot lying rather than as a narrowing. The guidance is
+ * generated from the tools passed to the configuration, so this is what proves the narrowed set
+ * is the one that got there.
+ */
+ expect(system).toContain("drive");
+ expect(system).not.toContain("slack: tool_0");
+ });
+});
+
+describe("a remote Bot", () => {
+ test("is sent the narrowed tools in its run body", async () => {
+ answerWith(["slack-digest"]);
+ const agents = await buildAgents(
+ [remoteAgent()],
+ model,
+ "test-key",
+ undefined,
+ async () => granted,
+ undefined,
+ undefined,
+ undefined,
+ selection(),
+ );
+ await ask(agents.risk as never, "summarise the Slack channel");
+
+ expect(sentToRemote).toHaveLength(1);
+ const offered = (sentToRemote[0]?.tools ?? []).filter((name) =>
+ name.startsWith("mcp__"),
+ );
+ expect(offered).toContain("mcp__slack__tool_0");
+ // Declared by the skill that was not chosen.
+ expect(offered).not.toContain("mcp__drive__tool_0");
+ // Undeclared, so it rides along on the remote path exactly as on the built-in one.
+ expect(offered).toContain("mcp__drive__tool_7");
+ });
+
+ test("still gets its standing role, its holdings and its signed run", async () => {
+ /*
+ * THIS IS THE TEST THAT CAUGHT THE REAL BUG. Narrowing was first built by wrapping the agent and
+ * delegating to `remote.run(input)`. Middleware registered with `.use()` is applied by
+ * `runAgent`, not by `run`, so the whole of `remoteAgentWithStandingRole` was skipped: the
+ * endpoint got a run with no role, no holdings, no tools and no signed assertion. Nothing threw.
+ * The Bot simply answered as though it had been told nothing, which is exactly what had
+ * happened.
+ */
+ answerWith(["slack-digest"]);
+ const agents = await buildAgents(
+ [remoteAgent()],
+ model,
+ "test-key",
+ undefined,
+ async () => granted,
+ () => "signed-assertion",
+ undefined,
+ undefined,
+ selection(),
+ );
+ await ask(agents.risk as never, "summarise the Slack channel");
+
+ const run = sentToRemote[0];
+ expect(run?.messages?.[0]?.id).toBe("standing-role:risk");
+ const holdings = (run?.messages ?? []).find(
+ (message) => message.id === "granted-tools:risk",
+ );
+ expect(String(holdings?.content ?? "")).toContain("slack");
+ // Narrowed away, so the Bot must not be told it holds it.
+ expect(String(holdings?.content ?? "")).not.toContain("drive: tool_0");
+ expect(run?.forwardedProps?.openbotBotId).toBe("risk");
+ expect(run?.forwardedProps?.openbotRun).toBe("signed-assertion");
+ // The deployment-run list has to be the narrowed set too, or the Bot is told this side executes
+ // a tool it was never offered.
+ expect(run?.forwardedProps?.openbotDeploymentTools).toContain(
+ "mcp__slack__tool_0",
+ );
+ expect(run?.forwardedProps?.openbotDeploymentTools).not.toContain(
+ "mcp__drive__tool_0",
+ );
+ });
+});
+
+describe("when selection cannot help", () => {
+ test("a catalogue under the floor is never sent to pass one", async () => {
+ llm.onMessage(/.*/, { type: "text", content: "Here is what I found." });
+ const few = granted.slice(0, 6);
+ const agents = await buildAgents(
+ [builtIn],
+ model,
+ "test-key",
+ undefined,
+ async () => few,
+ undefined,
+ undefined,
+ undefined,
+ selection(),
+ );
+ await ask(agents.analyst as never, "read the Drive doc");
+
+ const prompts = llm
+ .getRequests()
+ .flatMap((entry) =>
+ (
+ (entry.body as { messages?: { content?: unknown }[] })?.messages ?? []
+ ).map((message) => String(message.content ?? "")),
+ );
+ expect(
+ prompts.some((prompt) =>
+ prompt.includes("You choose which capabilities to load"),
+ ),
+ ).toBe(false);
+ expect(toolsOfferedToModel()).toHaveLength(few.length);
+ });
+
+ test("a Bot whose skills declare nothing is never sent to pass one", async () => {
+ llm.onMessage(/.*/, { type: "text", content: "Here is what I found." });
+ const agents = await buildAgents(
+ [builtIn],
+ model,
+ "test-key",
+ undefined,
+ async () => granted,
+ undefined,
+ undefined,
+ undefined,
+ {
+ loadSkills: async () => [
+ {
+ slug: "prose",
+ title: "Prose",
+ summary: "Instructions only",
+ tools: [],
+ },
+ ],
+ choose: async () => {
+ throw new Error("should not be asked");
+ },
+ },
+ );
+ await ask(agents.analyst as never, "read the Drive doc");
+ expect(toolsOfferedToModel()).toHaveLength(granted.length);
+ });
+
+ test("skills that cannot be read leave the Bot with all of its tools", async () => {
+ llm.onMessage(/.*/, { type: "text", content: "Here is what I found." });
+ const agents = await buildAgents(
+ [builtIn],
+ model,
+ "test-key",
+ undefined,
+ async () => granted,
+ undefined,
+ undefined,
+ undefined,
+ {
+ loadSkills: async () => {
+ throw new Error("database is down");
+ },
+ choose: async () => JSON.stringify({ skills: ["drive-audit"] }),
+ },
+ );
+ await ask(agents.analyst as never, "read the Drive doc");
+ expect(toolsOfferedToModel()).toHaveLength(granted.length);
+ });
+});
+
+describe("the discovery record", () => {
+ test("names the narrowing, and is written before the model is asked", async () => {
+ recorded.length = 0;
+ answerWith(["drive-audit"]);
+ const agents = await buildAgents(
+ [builtIn],
+ model,
+ "test-key",
+ undefined,
+ async () => granted,
+ undefined,
+ undefined,
+ undefined,
+ selection(),
+ );
+ await ask(agents.analyst as never, "read the Drive doc");
+
+ expect(recorded).toHaveLength(1);
+ const entry = recorded[0];
+ expect(entry?.reason).toBe("selected");
+ expect(entry?.skills).toEqual(["drive-audit"]);
+ expect(entry?.granted).toBe(granted.length);
+ expect(entry?.offered).toHaveLength(8);
+ });
+
+ test("a record that throws does not cost the run", async () => {
+ answerWith(["drive-audit"]);
+ const agents = await buildAgents(
+ [builtIn],
+ model,
+ "test-key",
+ undefined,
+ async () => granted,
+ undefined,
+ undefined,
+ undefined,
+ {
+ loadSkills: async () => skills,
+ choose: createModelCompleter({
+ model,
+ resolveApiKey: async () => "test-key",
+ }),
+ record: async () => {
+ throw new Error("audit table is gone");
+ },
+ },
+ );
+ // The assertion is that this resolves at all. An audit write is not worth a person's answer.
+ await ask(agents.analyst as never, "read the Drive doc");
+ expect(toolsOfferedToModel()).toHaveLength(8);
+ });
+});