Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 40 additions & 1 deletion agent-computer/src/control.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,15 @@ export type ControlState = {
reason?: string;
/** True once the Bot has asked for help and no person has taken the wheel yet. */
requested: boolean;
/**
* When the Bot asked, so an unanswered request can stop being shown.
*
* A request nobody answers used to last forever. The run that made it had already ended, but the
* prompt stayed on the computer, and control belongs to the computer rather than to a conversation
* — so every later conversation with that Bot showed a live "Take control" for work it was not
* doing, with the reason the Bot gave, written for whoever asked and rendered to whoever looked.
*/
requestedAt?: string;
/**
* A secret the Bot is waiting for, described by its label only.
*
Expand Down Expand Up @@ -57,6 +66,16 @@ export class ControlRequestError extends Error {
}

export const NO_SECRET_PENDING = "Nothing is waiting for a secret.";
/**
* How long an unanswered request to take the wheel is shown for.
*
* Long enough that somebody who stepped away can still act on it, short enough that it does not
* follow the Bot into tomorrow's conversations. The run that made it is already over either way:
* nothing resumes when a person takes the wheel this late, so the value trades "still useful" against
* "still on screen" and nothing else.
*/
export const HELP_REQUEST_TTL_MS = 10 * 60 * 1000;

export const HUMAN_HAS_CONTROL =
"A person has control of the computer right now. Wait for them to hand it back before acting.";
export const TAKE_CONTROL_FIRST =
Expand All @@ -79,8 +98,27 @@ export function createControl(
};

return {
/** The current state, as the surface polls it. A copy, so a caller cannot mutate the machine. */
/**
* The current state, as the surface polls it. A copy, so a caller cannot mutate the machine.
*
* An unanswered request is dropped once it is older than {@link HELP_REQUEST_TTL_MS}. It is
* expired on read rather than on a timer because there is nothing to wake: the run that asked
* has ended, and the only thing that cares is whoever looks next.
*
* Only ever the ASK. A person actually holding the wheel is never timed out from under them:
* they may be halfway through typing a code, and taking the browser back mid-sign-in is worse
* than any stale prompt.
*/
get(): ControlState {
if (
state.requested &&
state.holder === "bot" &&
state.requestedAt &&
Date.parse(now()) - Date.parse(state.requestedAt) > HELP_REQUEST_TTL_MS
) {
const { reason: _reason, requestedAt: _at, ...rest } = state;
state = { ...rest, requested: false };
}
return { ...state };
},

Expand All @@ -94,6 +132,7 @@ export function createControl(
state = {
...state,
requested: true,
requestedAt: now(),
reason:
typeof reason === "string" && reason.trim()
? reason.trim()
Expand Down
50 changes: 50 additions & 0 deletions agent-computer/tests/control.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -236,3 +236,53 @@ describe("the crappy paths: secrets", () => {
).toEqual(["secretRef", "secretSnapshotId", "secretWanted"]);
});
});

/**
* A request nobody answered does not outlive the run that made it.
*
* Control belongs to the computer, not to a conversation, and an unanswered request used to sit on
* it forever. The run that asked had ended, but every later conversation with that Bot showed a live
* "Take control" for work it was not doing — and showed the reason the Bot gave, which is written
* for whoever asked and was being rendered to whoever looked.
*
* Seen in the product: a brand new channel, on an unrelated question, displaying "Google Docs is
* asking for sign-in before I can read the PRD document" from a conversation minutes earlier.
*/
describe("an unanswered request to take the wheel", () => {
test("is still shown inside the window", () => {
let clock = "2026-08-22T03:00:00.000Z";
const control = createControl(() => clock);
control.requestHelp("sign in to Drive");

clock = "2026-08-22T03:05:00.000Z";
const state = control.get();
expect(state.requested).toBe(true);
expect(state.reason).toBe("sign in to Drive");
});

test("stops being shown once it is stale, and takes its reason with it", () => {
let clock = "2026-08-22T03:00:00.000Z";
const control = createControl(() => clock);
control.requestHelp("sign in to Drive");

clock = "2026-08-22T03:20:00.000Z";
const state = control.get();
expect(state.requested).toBe(false);
// The reason is the part that leaked between conversations, so it goes too.
expect(state.reason).toBeUndefined();
});

test("never takes the wheel back off a person who holds it", () => {
/*
* The one case that must not expire. Somebody may be halfway through typing a code, and pulling
* the browser back mid-sign-in is worse than any stale prompt. Only the ASK times out.
*/
let clock = "2026-08-22T03:00:00.000Z";
const control = createControl(() => clock);
control.requestHelp("sign in to Drive");
control.take();

clock = "2026-08-22T04:00:00.000Z";
expect(control.get().holder).toBe("human");
});
});
19 changes: 19 additions & 0 deletions server/src/app.ts
Original file line number Diff line number Diff line change
Expand Up @@ -659,6 +659,25 @@ export function createApp(
intentRouter,
requireUser,
auditStore,
/*
* Which vendors each coworker holds tools for, so the router weighs what a coworker can
* reach and not only what somebody wrote it was for. Only when there is a plugin store to
* ask: a deployment with no connectors routes exactly as it did.
*/
pluginStore
? async (agentId) => {
const granted = await pluginStore.listForAgent(agentId);
return [
...new Set(
granted.tools.map(
(tool) =>
tool.toolName.replace(/^mcp__/, "").split("__")[0] ??
tool.toolName,
),
),
];
}
: undefined,
),
);
}
Expand Down
42 changes: 38 additions & 4 deletions server/src/copilot.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import {
import { createCopilotHonoHandler } from "@copilotkit/runtime/v2/hono";
import { z } from "zod";
import { COMPUTER_GUIDANCE } from "../../shared/bot-prompt";
import { grantedToolGuidance } from "./plugins/tools";
import type { AgentActor } from "./agents/profile-types";
import type { StallGuard } from "./channels/stall-guard";
import type { DeploymentConfig } from "./config";
Expand Down Expand Up @@ -200,9 +201,18 @@ export function builtInAgentConfiguration(

return {
model: `${model.provider}/${model.defaultModel}`,
prompt: computerGuidance
? `${agent.systemPrompt}\n\n${computerGuidance}`
: agent.systemPrompt,
/*
* The package's role, then what this Bot actually holds, then the computer.
*
* The grants go BEFORE the computer prose on purpose. That prose is long and emphatic about the
* browser and mentions connectors nowhere, so a Bot that read it last reached for the browser
* even when it held a tool for the exact system being asked about.
*/
prompt: [
agent.systemPrompt,
...(grantedToolGuidance(tools) ? [grantedToolGuidance(tools)] : []),
...(computerGuidance ? [computerGuidance] : []),
].join("\n\n"),
apiKey,
/*
* A run stops after one step unless told otherwise, which for a Bot with tools means it calls
Expand Down Expand Up @@ -327,13 +337,37 @@ function remoteAgentWithStandingRole(
? { fetch: stallGuard.watch({ id: agent.id, name: agent.name }) }
: {}),
});
/*
* What this Bot holds, as a second standing message.
*
* Beside the role rather than inside it, because the role comes from the package and this comes
* from the grants: they change for different reasons and at different times. Sent on every run for
* the same reason the tools are, so switching a connector on reaches the next run.
*
* The remote path needs this more than the built-in one, not less. A framework Bot is handed the
* tools as an offer and decides for itself what to call, with `COMPUTER_GUIDANCE` as its whole
* prompt — a page about the browser that mentions connectors nowhere. That is the Bot that browsed
* to drive.google.com holding four Drive tools.
*/
const holdings = grantedToolGuidance(tools);
const holdingsMessage = holdings
? {
id: `granted-tools:${agent.id}`,
role: "system" as const,
content: holdings,
}
: null;

remote.use((input, next) =>
next.run({
...input,
messages: [
agent.standingMessage,
...(holdingsMessage ? [holdingsMessage] : []),
...input.messages.filter(
(message) => message.id !== agent.standingMessage.id,
(message) =>
message.id !== agent.standingMessage.id &&
message.id !== holdingsMessage?.id,
),
],
/*
Expand Down
42 changes: 42 additions & 0 deletions server/src/plugins/tools.ts
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,48 @@ export function parametersFor(inputSchema: Record<string, unknown>): z.ZodType {
return z.object({}).catchall(z.unknown());
}

/**
* What this Bot holds, said in its instructions rather than left to be inferred from a tool list.
*
* A tool array tells a model a tool exists. It does not tell it that the tool is the right way to
* reach that system, and it competes with a page of prose about the browser that every Bot is given
* whether or not it has any connectors at all. The browser prose wins: it is emphatic, it is about
* capability, and it says "never claim you cannot browse".
*
* So a Bot holding four Google Drive tools browsed to drive.google.com, met a sign-in page its
* container could never satisfy, and asked its person to sign in to a vendor that person had already
* connected. The tools were there the whole time.
*
* Generated from the grants rather than written down, because the point is that it tracks them. An
* administrator switching a connector on, or granting one more of its tools, changes what the Bot is
* told on its next run with nothing else to remember and nothing to keep in step.
*
* Empty when the Bot holds nothing, so a deployment with no connectors says nothing about them.
*/
export function grantedToolGuidance(tools: GrantedTool[]): string {
if (tools.length === 0) return "";

const bySystem = new Map<string, string[]>();
for (const tool of tools) {
// `mcp__server__tool`, which is the shape the model is offered.
const parts = tool.name.replace(/^mcp__/, "").split("__");
const system = parts.length > 1 ? (parts[0] as string) : "this deployment";
const rest = parts.length > 1 ? parts.slice(1).join("__") : tool.name;
bySystem.set(system, [...(bySystem.get(system) ?? []), rest]);
}

return [
"You can reach these systems directly, as the person asking, with their own access:",
...[...bySystem.entries()].map(
([system, names]) => `- ${system}: ${names.join(", ")}`,
),
"Use them for anything about those systems. Do NOT browse to one of their websites instead: your",
"browser is signed in as nobody, so it sees less than these tools do and will meet a sign-in wall",
"that connecting an account has already solved. If one of these systems is involved but no tool",
"above covers the part you need, say which part is missing rather than going around it.",
].join("\n");
}

/**
* Every MCP tool granted to one Bot, ready to hand to the runtime.
*
Expand Down
37 changes: 36 additions & 1 deletion server/src/routing/classify.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,20 @@ export type RoutingCandidate = {
name: string;
/** What this coworker is for. The one line an operator wrote to say when to reach them. */
roleDescription: string;
/**
* The systems this coworker can actually reach, by name.
*
* Routing on the role description alone routes on what somebody wrote a coworker was for, which is
* not the same as what it can do. A question about a document in Google Drive went to the coworker
* whose description says "company knowledge" and which held no Drive grants at all, so it browsed
* to the vendor, met a sign-in wall and asked the person to sign in to an account they had already
* connected. The coworker that could have answered was one line further down the roster.
*
* Empty for a coworker holding nothing, which is most of them in most deployments. It is a hint
* rather than a filter: a specialist with no connectors is still the right answer to a question
* about its specialism.
*/
reaches?: readonly string[];
};

export type RoutingDecision = {
Expand All @@ -37,7 +51,16 @@ export function routingPrompt(
candidates: readonly RoutingCandidate[],
): string {
const roster = candidates
.map((c) => `- id: ${c.id}\n name: ${c.name}\n for: ${c.roleDescription}`)
.map((c) =>
[
`- id: ${c.id}`,
` name: ${c.name}`,
` for: ${c.roleDescription}`,
...(c.reaches && c.reaches.length > 0
? [` can reach: ${c.reaches.join(", ")}`]
: []),
].join("\n"),
)
.join("\n");
return [
"You route a person's message to the one coworker best suited to it.",
Expand All @@ -46,6 +69,18 @@ export function routingPrompt(
"",
'Reply with only JSON: {"agentId": "<one id from the list>", "reason": "<short, names the fit>", "confidence": <0..1>}.',
"Pick the specialist whose purpose matches the message. If none clearly fits, use the most general coworker and give it a low confidence.",
/*
* Only when somebody on the roster can actually reach something.
*
* A deployment with no connectors would otherwise carry a rule about systems none of its
* coworkers have, in every routing prompt it ever sends. Same principle as the guidance a Bot
* gets about its own grants: say nothing about what is not there.
*/
...(candidates.some((c) => c.reaches && c.reaches.length > 0)
? [
"When the message names a system a coworker can reach, prefer that coworker: one that cannot reach it has no way to answer and will fall back to a browser that is signed in as nobody. Purpose still comes first — a specialist with no systems listed is right for a question about its specialism.",
]
: []),
"",
`Message: ${text}`,
].join("\n");
Expand Down
32 changes: 27 additions & 5 deletions server/src/routing/routes.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,14 @@ export function createRoutingRoutes(
router: IntentRouter,
requireUser: MiddlewareHandler<{ Variables: AppVariables }>,
auditStore?: AuditStore,
/**
* Which systems a coworker can reach, for the router to weigh alongside what it is for.
*
* Optional, and absent leaves routing exactly as it was: a deployment with no connectors has
* nothing to add here, and one that cannot answer the question should not have routing fail over
* it. Asked per request rather than held, because a grant added a minute ago has to count.
*/
reachableSystems?: (agentId: string) => Promise<readonly string[]>,
) {
const routes = new Hono<{ Variables: AppVariables }>();

Expand All @@ -39,11 +47,25 @@ export function createRoutingRoutes(
if (!preferred) {
return context.json({ error: "No coworker is available." }, 409);
}
const candidates: RoutingCandidate[] = roster.map((a) => ({
id: a.id,
name: a.name,
roleDescription: a.roleDescription,
}));
const candidates: RoutingCandidate[] = await Promise.all(
roster.map(async (a) => ({
id: a.id,
name: a.name,
roleDescription: a.roleDescription,
/*
* Never allowed to break routing. A connector store that is slow or unhappy must not turn
* "who is this for" into an error, so a failure here is the same as holding nothing: the
* router falls back to matching on purpose alone, which is what it did before.
*/
...(reachableSystems
? {
reaches: await reachableSystems(a.id).catch(
() => [] as readonly string[],
),
}
: {}),
})),
);

const decision = await router.route(text, candidates, preferred.id);

Expand Down
Loading