diff --git a/docs/configuration.md b/docs/configuration.md index 333424f7..95adf337 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -319,6 +319,16 @@ Each agent requires `id`, `name`, `title`, `role_description`, and `type`. | `built-in` | `system_prompt` | | `remote-ag-ui` | `endpoint` | +The two types are told different amounts, which is easy to miss. A `built-in` agent gets its +`system_prompt`; a `remote-ag-ui` agent has none, and its `role_description` is the only instruction +it ever receives from the package. Write that sentence as the whole brief for the Bot, not as a +label for a list. + +Both kinds are also told, by the deployment rather than by the package, to say where an answer came +from: cite what a tool returned, and say plainly when the answer is from the model's own knowledge +rather than from anything it read. That rule is not written per agent, so it cannot be missing from +the next one somebody adds. + Any `${NAME}` in a package file is replaced with that environment variable, so one package works against a local stack, a staging one and production. `${NAME:-fallback}` uses the fallback when the name is unset or empty, which is how the example package points at the Bot in the box without diff --git a/server/src/copilot.ts b/server/src/copilot.ts index 8582a85d..d6920468 100644 --- a/server/src/copilot.ts +++ b/server/src/copilot.ts @@ -7,7 +7,10 @@ import { } from "@copilotkit/runtime/v2"; import { createCopilotHonoHandler } from "@copilotkit/runtime/v2/hono"; import { z } from "zod"; -import { COMPUTER_GUIDANCE } from "../../shared/bot-prompt"; +import { + COMPUTER_GUIDANCE, + PROVENANCE_GUIDANCE, +} from "../../shared/bot-prompt"; import { grantedToolGuidance } from "./plugins/tools"; import type { AgentActor } from "./agents/profile-types"; import type { StallGuard } from "./channels/stall-guard"; @@ -97,6 +100,14 @@ export function standingRoleMessage( `You are ${profile.name}, ${profile.title}.`, profile.roleDescription, "This standing role applies in every channel. Treat channel messages as task-specific instructions within it.", + /* + * Here rather than in the package, because for a remote Bot the standing role is the only + * instruction there is: `role_description` is one sentence somebody wrote about what it is + * for, and nothing else reaches it. The compliance Bot that answered a filing question with + * thresholds and deadlines and no source was a `remote-ag-ui` agent whose entire prompt was + * "Investigate policies, transaction monitoring, and control evidence." + */ + PROVENANCE_GUIDANCE, ].join("\n\n"), }; } @@ -210,6 +221,14 @@ export function builtInAgentConfiguration( */ prompt: [ agent.systemPrompt, + /* + * Unconditional, unlike the two below it. + * + * Those describe things a deployment may or may not have. This describes how to answer at all, + * and a Bot with no tools and no computer needs it most: it has nothing to read, so everything + * it says comes from its own knowledge, and saying so is the only honest move available. + */ + PROVENANCE_GUIDANCE, ...(grantedToolGuidance(tools) ? [grantedToolGuidance(tools)] : []), ...(computerGuidance ? [computerGuidance] : []), ].join("\n\n"), diff --git a/server/tests/copilot.test.ts b/server/tests/copilot.test.ts index 87791b21..be3a4847 100644 --- a/server/tests/copilot.test.ts +++ b/server/tests/copilot.test.ts @@ -9,6 +9,7 @@ import { resolveRuntimeAgents, standingRoleMessage, } from "../src/copilot"; +import { PROVENANCE_GUIDANCE } from "../../shared/bot-prompt"; import { grantedToolGuidance } from "../src/plugins/tools"; // Every agent row now joins its profile, so the row a coworker is built from always names it. @@ -99,7 +100,9 @@ describe("registered Copilot agents", () => { ), ).toEqual({ model: "openai/gpt-4.1", - prompt: "Be helpful.", + // The provenance rule is unconditional, so even a Bot with no tools and no computer carries + // it. That Bot needs it most: nothing it says was read anywhere. + prompt: `Be helpful.\n\n${PROVENANCE_GUIDANCE}`, apiKey: "openai-secret", }); }); @@ -324,6 +327,10 @@ describe("standing agent roles", () => { "You are Expense Manager, Finance Operations.", "Review receipts, categorize expenses, and prepare reimbursement reports.", "This standing role applies in every channel. Treat channel messages as task-specific instructions within it.", + // For a remote Bot this message is the whole instruction, so the provenance rule has to + // travel in it or the Bot never hears it. Referenced rather than restated, so the assertion + // stays exact without pinning the wording twice. + PROVENANCE_GUIDANCE, ].join("\n\n"), }); }); @@ -444,6 +451,7 @@ describe("standing agent roles", () => { "You are Expense Manager, Finance Operations.", "Reconcile corporate card statements.", "This standing role applies in every channel. Treat channel messages as task-specific instructions within it.", + PROVENANCE_GUIDANCE, ].join("\n\n"), ); }); @@ -586,3 +594,76 @@ describe("what a Bot is told it holds", () => { ); }); }); + +/** + * Where an answer came from, on every Bot rather than the ones somebody remembered. + * + * Asked what the obligation was for twelve cash deposits under the reporting threshold, the + * compliance Bot answered with a filing requirement, a dollar threshold, a thirty-day deadline and a + * five-year retention period. The audit trail for that turn holds exactly one row: the routing + * decision. No tool call, no source, and nothing saying the answer came from the model. + * + * One package's `knowledge` Bot had a rule against this in its YAML. The Bot whose entire subject is + * regulatory obligation did not, because a `remote-ag-ui` agent gets its role description and + * nothing else. That asymmetry is the bug: a rule this important living in one agent's YAML is a + * rule the next agent will not have. + * + * So both paths are asserted, because they are built by different functions and a fix to one is not + * a fix to the other. + */ +describe("where a Bot says its answer came from", () => { + test("a built-in Bot carries the rule even holding nothing at all", () => { + // The Bot that needs it most. No tools and no computer means nothing it says was read anywhere. + const prompt = builtInAgentConfiguration( + { + id: "general-assistant", + name: "General Assistant", + type: "built_in", + systemPrompt: "Be helpful.", + }, + { provider: "openai", defaultModel: "gpt-4.1" }, + "openai-secret", + ).prompt as string; + + expect(prompt).toContain(PROVENANCE_GUIDANCE); + }); + + test("a remote Bot carries it in the only instruction it ever gets", () => { + const content = standingRoleMessage({ + id: "risk-analyst", + name: "Risk Analyst", + title: "Risk & Compliance", + roleDescription: + "Investigate policies, transaction monitoring, and control evidence.", + }).content; + + expect(content).toContain(PROVENANCE_GUIDANCE); + }); + + test("it does not send the Bot hunting for a source", () => { + /* + * The failure mode of the first attempt at this, which never left a branch. Told to find a + * source, Bots went reading the open web and looped on a government 404 page. An unsourced + * answer that says it is unsourced is honest; a search that never ends is a Bot that never + * answers. + */ + const guidance = PROVENANCE_GUIDANCE.toLowerCase().replace(/\s+/g, " "); + expect(guidance).toContain("this is not an instruction to go looking"); + expect(guidance).toContain("mark it plainly as unverified"); + expect(guidance).toContain("do not go hunting the open web"); + }); + + test("it names the answers that must not be stated without a source", () => { + // The general rule is easy to read past. The list is what makes it bite on the turn that + // produced this: a threshold, a deadline, a filing obligation, a figure. + const guidance = PROVENANCE_GUIDANCE.toLowerCase().replace(/\s+/g, " "); + for (const kind of [ + "threshold", + "deadline", + "filing obligation", + "figure", + ]) { + expect(guidance).toContain(kind); + } + }); +}); diff --git a/shared/bot-prompt.ts b/shared/bot-prompt.ts index 63897d0d..49a5c306 100644 --- a/shared/bot-prompt.ts +++ b/shared/bot-prompt.ts @@ -83,3 +83,57 @@ export const COMPUTER_GUIDANCE = COMPUTER_GUIDANCE_LINES.reduce( }, [""], ).join("\n\n"); + +/** + * Where an answer came from, said out loud. + * + * Asked "a customer made 12 cash deposits just under the reporting threshold, what is our + * obligation", the compliance Bot answered at length and with confidence: file a SAR, $5,000 or + * more, within 30 calendar days of initial detection, retain for 5 years. The audit trail for that + * turn holds one row, the routing decision. No tool call, no source, and no sentence anywhere saying + * the answer came from the model rather than from anything this deployment can reach. + * + * Several of those numbers may well be right, and that is the problem. A confident, plausible, + * unsourced answer is indistinguishable from a confident, plausible, wrong one, and nothing marked + * the difference on a question about whether to file, against what threshold, inside what deadline. + * + * One package's `knowledge` Bot had a rule against exactly this, written into its YAML by whoever + * happened to think of it. The Bot whose whole subject is regulatory obligation did not, because + * `remote-ag-ui` gets its role description and nothing else. A rule that important sitting in one + * agent's YAML is a rule that will be missing from the next agent somebody adds, so it lives here + * and every Bot gets it. + * + * The last paragraph is not padding. An earlier attempt at this told Bots to go and find a source, + * and they went hunting the open web and looped on a government 404 page, which is worse than the + * problem: an unsourced answer marked as unsourced is honest, and a hunt for one is a Bot that + * never answers. The instruction is to say where the answer came from, not to go looking. + */ +const PROVENANCE_GUIDANCE_LINES = [ + "Say where an answer came from. When you read it with one of your tools, cite what you read.", + "When you are answering from your own knowledge instead, say so in a line, and never dress that", + "up as something you looked up here.", + "", + "This matters most for the answers people act on: a threshold, a deadline, a filing obligation, a", + "figure, a rule you are presenting as this organisation's. Never state one of those as established", + "here without having read it somewhere you can name. Saying 'I have not checked this against your", + "own policy or the current regulation' costs you a sentence. Being confidently wrong about a", + "number somebody acts on costs them a great deal more.", + "", + "This is not an instruction to go looking. If nothing you can reach covers the question, answer as", + "well as you can and mark it plainly as unverified. Do not go hunting the open web for something", + "to cite, and do not keep retrying a page that is not giving you one: an unsourced answer that", + "says it is unsourced is honest, and a search that never ends is a Bot that never answers.", +]; + +export const PROVENANCE_GUIDANCE = PROVENANCE_GUIDANCE_LINES.reduce( + (paragraphs, line) => { + if (line === "") { + paragraphs.push(""); + return paragraphs; + } + const last = paragraphs.length - 1; + paragraphs[last] = paragraphs[last] ? `${paragraphs[last]} ${line}` : line; + return paragraphs; + }, + [""], +).join("\n\n");