From 4e5f329b9219ba52ec4b348ae92e3898c35cf92c Mon Sep 17 00:00:00 2001 From: David McKay Date: Fri, 21 Aug 2026 21:28:40 -0700 Subject: [PATCH] Make every Bot say where its answer came from Asked what the obligation was for twelve cash deposits under the reporting threshold, the compliance Bot answered at length: file a SAR, $5,000 or more, within 30 calendar days of initial detection, retain for 5 years. The audit trail for that turn holds one row, the routing decision. No tool call, no source, and no sentence saying the answer came from the model rather than from anything the deployment can reach. Several of those numbers may well be right. That is the problem. A confident, plausible, unsourced answer is indistinguishable from a confident, plausible, wrong one, on a question about whether to file, against what threshold, inside what deadline. One package's knowledge Bot had a rule against this, written into its YAML by whoever thought of it. The Bot whose entire subject is regulatory obligation did not, because a remote-ag-ui agent gets its role description and nothing else. That asymmetry is the bug: a rule this important living in one agent's YAML is a rule the next agent will not have. So it lives in the shared prompt and every Bot gets it, through the built-in prompt and through the standing role message that is a remote Bot's only instruction. The rule is to say where an answer came from, not to go and find one. An earlier attempt told Bots to find a source and they went hunting the open web and looped on a government 404 page, which is worse than the problem it was fixing. An unsourced answer that says it is unsourced is honest; a search that never ends is a Bot that never answers. Also says in the configuration docs that role_description is the whole brief for a remote agent, which is not obvious beside a built-in one that gets a prompt. Closes #142. --- docs/configuration.md | 10 +++++ server/src/copilot.ts | 21 ++++++++- server/tests/copilot.test.ts | 83 +++++++++++++++++++++++++++++++++++- shared/bot-prompt.ts | 54 +++++++++++++++++++++++ 4 files changed, 166 insertions(+), 2 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 333424f7..95adf337 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -319,6 +319,16 @@ Each agent requires `id`, `name`, `title`, `role_description`, and `type`. | `built-in` | `system_prompt` | | `remote-ag-ui` | `endpoint` | +The two types are told different amounts, which is easy to miss. A `built-in` agent gets its +`system_prompt`; a `remote-ag-ui` agent has none, and its `role_description` is the only instruction +it ever receives from the package. Write that sentence as the whole brief for the Bot, not as a +label for a list. + +Both kinds are also told, by the deployment rather than by the package, to say where an answer came +from: cite what a tool returned, and say plainly when the answer is from the model's own knowledge +rather than from anything it read. That rule is not written per agent, so it cannot be missing from +the next one somebody adds. + Any `${NAME}` in a package file is replaced with that environment variable, so one package works against a local stack, a staging one and production. `${NAME:-fallback}` uses the fallback when the name is unset or empty, which is how the example package points at the Bot in the box without diff --git a/server/src/copilot.ts b/server/src/copilot.ts index 8582a85d..d6920468 100644 --- a/server/src/copilot.ts +++ b/server/src/copilot.ts @@ -7,7 +7,10 @@ import { } from "@copilotkit/runtime/v2"; import { createCopilotHonoHandler } from "@copilotkit/runtime/v2/hono"; import { z } from "zod"; -import { COMPUTER_GUIDANCE } from "../../shared/bot-prompt"; +import { + COMPUTER_GUIDANCE, + PROVENANCE_GUIDANCE, +} from "../../shared/bot-prompt"; import { grantedToolGuidance } from "./plugins/tools"; import type { AgentActor } from "./agents/profile-types"; import type { StallGuard } from "./channels/stall-guard"; @@ -97,6 +100,14 @@ export function standingRoleMessage( `You are ${profile.name}, ${profile.title}.`, profile.roleDescription, "This standing role applies in every channel. Treat channel messages as task-specific instructions within it.", + /* + * Here rather than in the package, because for a remote Bot the standing role is the only + * instruction there is: `role_description` is one sentence somebody wrote about what it is + * for, and nothing else reaches it. The compliance Bot that answered a filing question with + * thresholds and deadlines and no source was a `remote-ag-ui` agent whose entire prompt was + * "Investigate policies, transaction monitoring, and control evidence." + */ + PROVENANCE_GUIDANCE, ].join("\n\n"), }; } @@ -210,6 +221,14 @@ export function builtInAgentConfiguration( */ prompt: [ agent.systemPrompt, + /* + * Unconditional, unlike the two below it. + * + * Those describe things a deployment may or may not have. This describes how to answer at all, + * and a Bot with no tools and no computer needs it most: it has nothing to read, so everything + * it says comes from its own knowledge, and saying so is the only honest move available. + */ + PROVENANCE_GUIDANCE, ...(grantedToolGuidance(tools) ? [grantedToolGuidance(tools)] : []), ...(computerGuidance ? [computerGuidance] : []), ].join("\n\n"), diff --git a/server/tests/copilot.test.ts b/server/tests/copilot.test.ts index 87791b21..be3a4847 100644 --- a/server/tests/copilot.test.ts +++ b/server/tests/copilot.test.ts @@ -9,6 +9,7 @@ import { resolveRuntimeAgents, standingRoleMessage, } from "../src/copilot"; +import { PROVENANCE_GUIDANCE } from "../../shared/bot-prompt"; import { grantedToolGuidance } from "../src/plugins/tools"; // Every agent row now joins its profile, so the row a coworker is built from always names it. @@ -99,7 +100,9 @@ describe("registered Copilot agents", () => { ), ).toEqual({ model: "openai/gpt-4.1", - prompt: "Be helpful.", + // The provenance rule is unconditional, so even a Bot with no tools and no computer carries + // it. That Bot needs it most: nothing it says was read anywhere. + prompt: `Be helpful.\n\n${PROVENANCE_GUIDANCE}`, apiKey: "openai-secret", }); }); @@ -324,6 +327,10 @@ describe("standing agent roles", () => { "You are Expense Manager, Finance Operations.", "Review receipts, categorize expenses, and prepare reimbursement reports.", "This standing role applies in every channel. Treat channel messages as task-specific instructions within it.", + // For a remote Bot this message is the whole instruction, so the provenance rule has to + // travel in it or the Bot never hears it. Referenced rather than restated, so the assertion + // stays exact without pinning the wording twice. + PROVENANCE_GUIDANCE, ].join("\n\n"), }); }); @@ -444,6 +451,7 @@ describe("standing agent roles", () => { "You are Expense Manager, Finance Operations.", "Reconcile corporate card statements.", "This standing role applies in every channel. Treat channel messages as task-specific instructions within it.", + PROVENANCE_GUIDANCE, ].join("\n\n"), ); }); @@ -586,3 +594,76 @@ describe("what a Bot is told it holds", () => { ); }); }); + +/** + * Where an answer came from, on every Bot rather than the ones somebody remembered. + * + * Asked what the obligation was for twelve cash deposits under the reporting threshold, the + * compliance Bot answered with a filing requirement, a dollar threshold, a thirty-day deadline and a + * five-year retention period. The audit trail for that turn holds exactly one row: the routing + * decision. No tool call, no source, and nothing saying the answer came from the model. + * + * One package's `knowledge` Bot had a rule against this in its YAML. The Bot whose entire subject is + * regulatory obligation did not, because a `remote-ag-ui` agent gets its role description and + * nothing else. That asymmetry is the bug: a rule this important living in one agent's YAML is a + * rule the next agent will not have. + * + * So both paths are asserted, because they are built by different functions and a fix to one is not + * a fix to the other. + */ +describe("where a Bot says its answer came from", () => { + test("a built-in Bot carries the rule even holding nothing at all", () => { + // The Bot that needs it most. No tools and no computer means nothing it says was read anywhere. + const prompt = builtInAgentConfiguration( + { + id: "general-assistant", + name: "General Assistant", + type: "built_in", + systemPrompt: "Be helpful.", + }, + { provider: "openai", defaultModel: "gpt-4.1" }, + "openai-secret", + ).prompt as string; + + expect(prompt).toContain(PROVENANCE_GUIDANCE); + }); + + test("a remote Bot carries it in the only instruction it ever gets", () => { + const content = standingRoleMessage({ + id: "risk-analyst", + name: "Risk Analyst", + title: "Risk & Compliance", + roleDescription: + "Investigate policies, transaction monitoring, and control evidence.", + }).content; + + expect(content).toContain(PROVENANCE_GUIDANCE); + }); + + test("it does not send the Bot hunting for a source", () => { + /* + * The failure mode of the first attempt at this, which never left a branch. Told to find a + * source, Bots went reading the open web and looped on a government 404 page. An unsourced + * answer that says it is unsourced is honest; a search that never ends is a Bot that never + * answers. + */ + const guidance = PROVENANCE_GUIDANCE.toLowerCase().replace(/\s+/g, " "); + expect(guidance).toContain("this is not an instruction to go looking"); + expect(guidance).toContain("mark it plainly as unverified"); + expect(guidance).toContain("do not go hunting the open web"); + }); + + test("it names the answers that must not be stated without a source", () => { + // The general rule is easy to read past. The list is what makes it bite on the turn that + // produced this: a threshold, a deadline, a filing obligation, a figure. + const guidance = PROVENANCE_GUIDANCE.toLowerCase().replace(/\s+/g, " "); + for (const kind of [ + "threshold", + "deadline", + "filing obligation", + "figure", + ]) { + expect(guidance).toContain(kind); + } + }); +}); diff --git a/shared/bot-prompt.ts b/shared/bot-prompt.ts index 63897d0d..49a5c306 100644 --- a/shared/bot-prompt.ts +++ b/shared/bot-prompt.ts @@ -83,3 +83,57 @@ export const COMPUTER_GUIDANCE = COMPUTER_GUIDANCE_LINES.reduce( }, [""], ).join("\n\n"); + +/** + * Where an answer came from, said out loud. + * + * Asked "a customer made 12 cash deposits just under the reporting threshold, what is our + * obligation", the compliance Bot answered at length and with confidence: file a SAR, $5,000 or + * more, within 30 calendar days of initial detection, retain for 5 years. The audit trail for that + * turn holds one row, the routing decision. No tool call, no source, and no sentence anywhere saying + * the answer came from the model rather than from anything this deployment can reach. + * + * Several of those numbers may well be right, and that is the problem. A confident, plausible, + * unsourced answer is indistinguishable from a confident, plausible, wrong one, and nothing marked + * the difference on a question about whether to file, against what threshold, inside what deadline. + * + * One package's `knowledge` Bot had a rule against exactly this, written into its YAML by whoever + * happened to think of it. The Bot whose whole subject is regulatory obligation did not, because + * `remote-ag-ui` gets its role description and nothing else. A rule that important sitting in one + * agent's YAML is a rule that will be missing from the next agent somebody adds, so it lives here + * and every Bot gets it. + * + * The last paragraph is not padding. An earlier attempt at this told Bots to go and find a source, + * and they went hunting the open web and looped on a government 404 page, which is worse than the + * problem: an unsourced answer marked as unsourced is honest, and a hunt for one is a Bot that + * never answers. The instruction is to say where the answer came from, not to go looking. + */ +const PROVENANCE_GUIDANCE_LINES = [ + "Say where an answer came from. When you read it with one of your tools, cite what you read.", + "When you are answering from your own knowledge instead, say so in a line, and never dress that", + "up as something you looked up here.", + "", + "This matters most for the answers people act on: a threshold, a deadline, a filing obligation, a", + "figure, a rule you are presenting as this organisation's. Never state one of those as established", + "here without having read it somewhere you can name. Saying 'I have not checked this against your", + "own policy or the current regulation' costs you a sentence. Being confidently wrong about a", + "number somebody acts on costs them a great deal more.", + "", + "This is not an instruction to go looking. If nothing you can reach covers the question, answer as", + "well as you can and mark it plainly as unverified. Do not go hunting the open web for something", + "to cite, and do not keep retrying a page that is not giving you one: an unsourced answer that", + "says it is unsourced is honest, and a search that never ends is a Bot that never answers.", +]; + +export const PROVENANCE_GUIDANCE = PROVENANCE_GUIDANCE_LINES.reduce( + (paragraphs, line) => { + if (line === "") { + paragraphs.push(""); + return paragraphs; + } + const last = paragraphs.length - 1; + paragraphs[last] = paragraphs[last] ? `${paragraphs[last]} ${line}` : line; + return paragraphs; + }, + [""], +).join("\n\n");