From a1bb1310528e9338043941eb7122d12a89e400c1 Mon Sep 17 00:00:00 2001 From: Ayal Kleinman Date: Mon, 24 Aug 2026 22:11:53 -0700 Subject: [PATCH 01/18] fix: name the element in a browser refusal, neutral mcp notwithstanding MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every browser context carries a neutral all-empty `mcp` object so a rule naming `mcp.effect` evaluates to false instead of throwing on an unbound identifier. The refusal copy keyed on that object being present rather than on its contents, so every live browser refusal took the tool-call branch and read ": on is blocked" — two empty strings where the element and the page belonged. The tests asserted the right sentence and passed anyway, because their contexts omitted the field the gateway always attaches. The new cases build the context the gateway actually builds. The branch now keys on the server and tool being named, which a real tool call always has and a browser action never does. --- server/src/computer/policy.ts | 7 +++- server/tests/computer-policy.test.ts | 50 ++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 1 deletion(-) diff --git a/server/src/computer/policy.ts b/server/src/computer/policy.ts index bc9df1cc..1766ecc4 100644 --- a/server/src/computer/policy.ts +++ b/server/src/computer/policy.ts @@ -318,7 +318,12 @@ function describeRefusal(context: PolicyContext, expression: string): string { ); } - if (context.mcp) { + // Present is not enough: the gateway attaches a neutral all-empty `mcp` to every browser context + // so a rule naming `mcp.effect` evaluates to false instead of throwing. Testing the object rather + // than its contents made this branch fire for every browser refusal, and a person whose click was + // refused read ": on is blocked" — two empty strings where the element and page belonged. A real + // tool call always names its server and its tool, so those are what the branch keys on. + if (context.mcp?.server || context.mcp?.tool) { return ( `This deployment's policy does not allow that: ${context.mcp.tool} on ` + `${context.mcp.server} is blocked by the rule \`${expression}\`.` diff --git a/server/tests/computer-policy.test.ts b/server/tests/computer-policy.test.ts index 2f834927..60c2462d 100644 --- a/server/tests/computer-policy.test.ts +++ b/server/tests/computer-policy.test.ts @@ -619,3 +619,53 @@ describe("a rule about one surface does not refuse another", () => { expect(decision.source).toBe("deny"); }); }); + + +describe("refusal wording under the context the gateway actually builds", () => { + /* + * The gateway attaches a neutral all-empty `mcp` to every browser context so a rule naming + * `mcp.effect` evaluates instead of throwing. The refusal copy used to key on the object being + * present rather than on its contents, so every live browser refusal read ": on is blocked" — + * empty strings where the element and page belonged — while tests that omitted the field passed. + */ + test("a browser refusal names the element, neutral mcp notwithstanding", () => { + const decision = evaluateActionPolicy( + { mode: "enforce", deny: ['contains(element.name, "Submit")'], allow: ["true"] }, + { + tool: { name: "computer_click" }, + bot: { id: "general-assistant" }, + actor: { id: "user:dev" }, + page: { url: "https://example.com/checkout", host: "example.com" }, + intent: "activate", + key: "", + element: { ref: "e1", role: "button", name: "Submit order", type: "" }, + file: { path: "", name: "", extension: "" }, + command: "", + mcp: { server: "", tool: "", effect: "" }, + }, + ); + expect(decision.allowed).toBe(false); + expect(decision.reason).toContain("Submit order"); + expect(decision.reason).toContain("example.com"); + expect(decision.reason).not.toContain(" on is blocked"); + }); + + test("a real tool call still reads as its server and tool", () => { + const decision = evaluateActionPolicy( + { mode: "enforce", deny: ['mcp.server == "notes"'], allow: ["true"] }, + { + tool: { name: "mcp__notes__search_notes" }, + bot: { id: "general-assistant" }, + actor: { id: "user:dev" }, + page: { url: "", host: "" }, + key: "", + element: { ref: "", role: "", name: "", type: "" }, + file: { path: "", name: "", extension: "" }, + command: "", + mcp: { server: "notes", tool: "search_notes", effect: "read" }, + }, + ); + expect(decision.allowed).toBe(false); + expect(decision.reason).toContain("search_notes on notes"); + }); +}); From 6b026c203abb039f2d6934273556e630a28a5fdd Mon Sep 17 00:00:00 2001 From: Ayal Kleinman Date: Mon, 24 Aug 2026 22:11:53 -0700 Subject: [PATCH 02/18] feat: test a boundary rule against history before saving it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A boundary was written blind: an administrator typed a CEL rule, saved it, and learned what it matches from the refusals it produced in production. The trail already records every judged computer action with the same facts the gateway judged it on, so the question "what would this rule have done" had an answer nobody could ask. `POST /policy-dry-run` takes a candidate policy, validates it exactly as PUT does, replays it over recent judged actions, and names each one it would have decided differently and the rule that would decide it. It writes nothing — not the policy, and no audit row, because no action is permitted or refused. The route is a new single segment reserved in DEPLOYMENT_ROUTES, which is the mechanism that file exists to make a third route think about. The replay is a reconstruction, not a simulation: `contextFromAuditPayload` rebuilds the gateway's context field for field from what the gateway recorded, through the same helpers (`intentOf`, `describeFile`, `hostOf`, now exported), so a rule behaves in the test as it will behave live. Absent facts become the same neutral empty strings the gateway uses, for the same reason: a replay that threw on `command` in a click would report a boundary far stricter than the one proposed. Rows that predate a field are skipped rather than guessed at. On the Boundaries page, Test first sits beside Add rule and runs the current policy plus the draft. The reply leads with the count over everything scanned, because the list under it is capped and a reader who stops at the rows should not believe the rows are the whole answer. --- CHANGELOG.md | 28 ++++ app/src/lib/computers/queries.ts | 41 +++++ app/src/routes/_authed/admin/boundaries.tsx | 107 ++++++++++++ server/src/app.ts | 1 + server/src/computer/deployment-routes.ts | 2 +- server/src/computer/gateway.ts | 6 +- server/src/computer/policy-dry-run.ts | 177 ++++++++++++++++++++ server/src/computer/routes.ts | 62 +++++++ server/tests/policy-dry-run.test.ts | 171 +++++++++++++++++++ 9 files changed, 591 insertions(+), 4 deletions(-) create mode 100644 server/src/computer/policy-dry-run.ts create mode 100644 server/tests/policy-dry-run.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index cbee9e84..7c13c147 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,34 @@ Newest first. `Unreleased` is what is on `main` and not yet tagged. ## Unreleased +### A rule can be tested against history before it is saved + +A boundary was written blind: an administrator typed a CEL rule, saved it, and learned what it +actually matches from the refusals it produced in production. The trail already records every judged +computer action with the same facts the gateway judged it on, so the question "what would this rule +have done" had an answer nobody could ask. + +The Boundaries page now has **Test first** beside **Add rule**. The candidate — the current policy +plus the drafted rule — is replayed over recent recorded actions, and the reply names each one it +would have decided differently and the rule that would have decided it. Nothing is saved and nothing +is decided; no audit row is written, because no action was permitted or refused. + +Replay, not simulation: the context is rebuilt from the audit row exactly as the gateway built it at +decision time, through the same helpers, so a rule behaves here as it will behave live. The scan is +bounded and biased to recency, and the reply says how many rows it covered. + +### A browser refusal names the element again + +Every browser context carries a neutral all-empty `mcp` object, so a rule naming `mcp.effect` +evaluates to false instead of throwing. The refusal copy keyed on that object being present rather +than on its contents, so every live browser refusal took the tool-call branch and read +": on is blocked" — two empty strings where the element and the page belonged. The tests passed, +because their contexts omitted the field the gateway always attaches. + +The branch now keys on the server and tool being named, which a real tool call always has. A refused +click reads "“Submit order” on shop.example is blocked by the rule ..." again, which is what the Bot +relays to the person asking. + ### Knowledge searches instead of guessing A package can say which of its skills each coworker gets, and the fintech example gives Knowledge the diff --git a/app/src/lib/computers/queries.ts b/app/src/lib/computers/queries.ts index 84cf9c09..ba632b5e 100644 --- a/app/src/lib/computers/queries.ts +++ b/app/src/lib/computers/queries.ts @@ -71,3 +71,44 @@ export function actionPolicyQueryOptions() { }), }); } + +/** One recorded action a candidate policy would decide differently than what happened. */ +export type DryRunChange = { + id: string; + createdAt: string; + action: string; + bot: string; + page: string; + element: { role: string; name: string } | null; + command: string | null; + file: string | null; + was: "allowed" | "refused"; + would: "allowed" | "refused"; + rule: string | null; + reason: string; +}; + +export type DryRunReport = { + scanned: number; + wouldRefuse: number; + wouldAllow: number; + unchanged: number; + /** Capped by the server; the counts cover everything scanned. */ + changes: DryRunChange[]; +}; + +/** + * What would this policy have decided, about recent recorded actions? + * + * A plain function rather than a query: the answer is about this candidate at this moment, nothing + * caches it and nothing invalidates it. It writes nothing — not the policy, and no audit row. + */ +export async function dryRunActionPolicy( + candidate: ActionPolicy, +): Promise { + return client("/api/computers/policy-dry-run", "report", { + method: "POST", + body: { policy: candidate }, + fallback: "The rule could not be tested against history.", + }); +} diff --git a/app/src/routes/_authed/admin/boundaries.tsx b/app/src/routes/_authed/admin/boundaries.tsx index cb6faca6..a4830562 100644 --- a/app/src/routes/_authed/admin/boundaries.tsx +++ b/app/src/routes/_authed/admin/boundaries.tsx @@ -6,6 +6,8 @@ import { saveActionPolicyMutationOptions } from "@/lib/computers/mutations"; import { type ActionPolicy, actionPolicyQueryOptions, + type DryRunReport, + dryRunActionPolicy, type PolicyMode, } from "@/lib/computers/queries"; import { queryClient } from "@/query-client"; @@ -51,6 +53,12 @@ function BoundariesPage() { const [saved, setSaved] = useState(false); const [draft, setDraft] = useState(""); + const [tested, setTested] = useState<{ + rule: string; + report: DryRunReport; + } | null>(null); + const [testing, setTesting] = useState(false); + const stored = useQuery(actionPolicyQueryOptions()); const savePolicy = useMutation(saveActionPolicyMutationOptions(queryClient)); @@ -90,6 +98,31 @@ function BoundariesPage() { if (!trimmed || policy.deny.includes(trimmed)) return; void save({ ...policy, deny: [...policy.deny, trimmed] }); setDraft(""); + setTested(null); + }; + + /* + * The rule as it would be in force — the current policy plus this draft — replayed over recent + * recorded actions. Nothing is saved and nothing is decided; the reply names the actions the + * addition would have decided differently, so the rule's real reach is known before it starts + * refusing anybody. + */ + const testRule = async (rule: string) => { + const trimmed = rule.trim(); + if (!trimmed) return; + setProblem(null); + setTesting(true); + try { + const report = await dryRunActionPolicy({ + ...policy, + deny: [...policy.deny, trimmed], + }); + setTested({ rule: trimmed, report }); + } catch (thrown) { + setProblem((thrown as Error).message); + } finally { + setTesting(false); + } }; return ( @@ -191,6 +224,7 @@ function BoundariesPage() { onChange={(event) => { setDraft(event.target.value); setSaved(false); + setTested(null); }} onKeyDown={(event) => { if (event.key === "Enter") addRule(draft); @@ -198,6 +232,14 @@ function BoundariesPage() { placeholder='tool.name == "computer_click" && contains(element.name, "submit")' value={draft} /> + + {tested ? : null} +