diff --git a/packages/agent-tools/src/desktop/canonical-tool-policy.ts b/packages/agent-tools/src/desktop/canonical-tool-policy.ts index 9a7844c40..5ac4720ce 100644 --- a/packages/agent-tools/src/desktop/canonical-tool-policy.ts +++ b/packages/agent-tools/src/desktop/canonical-tool-policy.ts @@ -20,6 +20,15 @@ const READ_ONLY_CANONICAL_BLOCKED_TOOL_NAMES = new Set([ 'request_feature_enable', ]); +/** + * Whether the canonical read-only ceiling (Explore/Verifier) removes this tool. + * Exposed so a runtime guard can treat a call that still arrives as a violation + * without keeping its own copy of the list. + */ +export function isReadOnlyCanonicalBlockedToolName(name: string): boolean { + return READ_ONLY_CANONICAL_BLOCKED_TOOL_NAMES.has(name) || isComputerUseRuntimeToolName(name); +} + /** Computer-use tools share the desktop_* namespace across native and MCP paths. */ export function isComputerUseRuntimeToolName(name: string): boolean { return name.startsWith('desktop_'); diff --git a/packages/agent-tools/src/index.ts b/packages/agent-tools/src/index.ts index 5a528e640..21c32c054 100644 --- a/packages/agent-tools/src/index.ts +++ b/packages/agent-tools/src/index.ts @@ -38,6 +38,7 @@ export { filterCanonicalBuiltinMcpEntries, filterCanonicalNativeToolCeiling, isCanonicalBuiltinTurn, + isReadOnlyCanonicalBlockedToolName, } from "./desktop/canonical-tool-policy.js"; export { getLocalBashEnvironment } from "./desktop/local-pi-tools.js"; diff --git a/packages/config/src/goal-config.ts b/packages/config/src/goal-config.ts index ffd3c5e63..7ebed92aa 100644 --- a/packages/config/src/goal-config.ts +++ b/packages/config/src/goal-config.ts @@ -6,6 +6,12 @@ export type GoalVerificationEvidenceMode = (typeof GOAL_VERIFICATION_EVIDENCE_MO export const GOAL_EVALUATOR_MODEL_POLICIES = ['same-route-small-fast'] as const; export const GOAL_VERIFIER_READONLY_PROFILE = 'goal-verifier-readonly'; export const GOAL_SUBAGENT_PROFILES = [GOAL_VERIFIER_READONLY_PROFILE] as const; +/** + * Network tools a Goal verifier child never receives. Verification adjudicates + * the workspace from local evidence, so these are withheld from the child's tool + * catalog, and the child's runtime guard refuses any call that still arrives. + */ +export const GOAL_VERIFIER_OFFLINE_TOOL_NAMES: readonly string[] = ['web_fetch', 'web_search']; export interface GoalConfig { /** Optional objective length cap. Absent means unlimited. */ diff --git a/packages/config/src/index.ts b/packages/config/src/index.ts index b9cac3d35..c325d980a 100644 --- a/packages/config/src/index.ts +++ b/packages/config/src/index.ts @@ -39,6 +39,7 @@ export { GOAL_VERIFICATION_EVIDENCE_MODES, GOAL_SUBAGENT_PROFILES, GOAL_VERIFICATION_MODES, + GOAL_VERIFIER_OFFLINE_TOOL_NAMES, GOAL_VERIFIER_READONLY_PROFILE, parseGoalConfig, } from './goal-config.js'; diff --git a/packages/local-runtime-v2/src/application/agent/goal-subagent-coordinator.ts b/packages/local-runtime-v2/src/application/agent/goal-subagent-coordinator.ts index 6382a1cbc..a9a77cc62 100644 --- a/packages/local-runtime-v2/src/application/agent/goal-subagent-coordinator.ts +++ b/packages/local-runtime-v2/src/application/agent/goal-subagent-coordinator.ts @@ -1,6 +1,7 @@ import type { AssistantMessage } from '@earendil-works/pi-ai'; -import { GOAL_VERIFIER_READONLY_PROFILE } from '@mavis/config'; +import { GOAL_VERIFIER_OFFLINE_TOOL_NAMES, GOAL_VERIFIER_READONLY_PROFILE } from '@mavis/config'; import type { PiBeforeToolCallHook } from '@mavis/agent-core/pi-turn-runner'; +import { isReadOnlyCanonicalBlockedToolName } from '@mavis/agent-tools'; import type { AgentExtension } from '@mavis/agent-runtime'; import type { VerificationHostContext } from '@mavis/goal'; @@ -8,13 +9,33 @@ import { summarizeCommittedPiGoalUsage } from '../../service/session-system/inde import { renderGoalVerifierReminder } from './goal-verifier-reminder.js'; /** - * The only tools the builtin `verifier` role holds that a Goal verifier child - * must not have. Everything else it could reach — write, edit, todowrite, the - * whole `task*` family, memory, computer use — is already gone by the time a - * call reaches this guard: the role asset grants five tools and the canonical - * read-only ceiling subtracts the rest. + * The network tools the builtin `verifier` role holds that a Goal verifier + * child must not use. The child's tool catalog already withholds them; this set + * is the fallback for a call that still arrives, for example a name the model + * remembers from its role prompt. + * + * A refused network call is a recoverable model mistake: the child still holds + * `read`, `grep`, `glob` and `bash` and can reach the same evidence locally. + * Everything else outside the role's read-only ceiling — write, edit, + * todowrite, delegation, memory, computer use — is a hard violation instead + * and stops the run on its first call. + */ +const GOAL_VERIFIER_OFFLINE_TOOLS: ReadonlySet = new Set(GOAL_VERIFIER_OFFLINE_TOOL_NAMES); + +/** + * Consecutive child turns that may reach for a network tool before the run is + * stopped as a capability violation. Counting turns rather than calls keeps + * several parallel calls from one model response to a single mistake; a child + * that is refused and then tries again on its very next turn is looping. */ -const GOAL_VERIFIER_DENIED_TOOLS = new Set(['web_fetch', 'web_search']); +const MAX_CONSECUTIVE_OFFLINE_TOOL_TURNS = 2; + +/** + * Child turns in total that may reach for a network tool. Bounds a child that + * alternates refused network calls with local work, independently of the + * optional host turn cap. + */ +const MAX_OFFLINE_TOOL_TURNS = 3; /** Structurally mirrors the v1 delegation runner's registry contract. */ export interface GoalVerifierChildRunIssue { @@ -39,6 +60,12 @@ export interface GoalVerifierChildRunState { childTurns: number; tokens: number; usageIncomplete: boolean; + /** Child turn in which a network tool call was last refused. */ + lastOfflineToolTurn?: number; + /** Consecutive child turns, ending at `lastOfflineToolTurn`, that called a network tool. */ + consecutiveOfflineToolTurns: number; + /** Child turns in total that called a network tool. */ + offlineToolTurns: number; issue?: GoalVerifierChildRunIssue; } @@ -47,7 +74,7 @@ export interface GoalVerifierChildRunState { * * The child runs on the ordinary delegation path under the builtin `verifier` * role, so this holds no authority: a Turn that claims a run id gains a - * reminder, a two-tool denylist, and a hard turn/token ceiling — all of them + * reminder, a narrow tool guard, and a hard turn/token ceiling — all of them * restrictions. There is nothing here worth forging. * * The ceiling still has to live somewhere: `pi-turn-runner` has no generic @@ -69,6 +96,8 @@ export class GoalVerifierChildCoordinator { childTurns: 0, tokens: 0, usageIncomplete: false, + consecutiveOfflineToolTurns: 0, + offlineToolTurns: 0, }); } @@ -159,7 +188,8 @@ export class GoalVerifierChildCoordinator { }; } const toolName = toolContext.toolCall.name; - if (!GOAL_VERIFIER_DENIED_TOOLS.has(toolName)) return undefined; + if (GOAL_VERIFIER_OFFLINE_TOOLS.has(toolName)) return this.refuseOfflineTool(state, toolName); + if (!isReadOnlyCanonicalBlockedToolName(toolName)) return undefined; const reason = `GOAL_VERIFIER_CAPABILITY_VIOLATION: readonly Goal verification blocked tool "${toolName}".`; // Latched, not aborted here: the blocked result goes back to the model and // the next `beforeLlm` stops the run on the latched issue. @@ -167,6 +197,42 @@ export class GoalVerifierChildCoordinator { return { block: true, reason }; } + /** + * Refuses a network tool call and tells the child how to continue offline. + * + * Only a child that keeps reaching for the network after being refused is a + * capability violation; every call made in one model response shares that + * response's turn, so parallel calls count once. + */ + private refuseOfflineTool( + state: GoalVerifierChildRunState, + toolName: string, + ): { readonly block: true; readonly reason: string } { + const turn = state.childTurns; + if (state.lastOfflineToolTurn !== turn) { + state.consecutiveOfflineToolTurns = + state.lastOfflineToolTurn === turn - 1 ? state.consecutiveOfflineToolTurns + 1 : 1; + state.offlineToolTurns += 1; + state.lastOfflineToolTurn = turn; + } + if ( + state.consecutiveOfflineToolTurns >= MAX_CONSECUTIVE_OFFLINE_TOOL_TURNS || + state.offlineToolTurns >= MAX_OFFLINE_TOOL_TURNS + ) { + const reason = `GOAL_VERIFIER_CAPABILITY_VIOLATION: readonly Goal verification kept calling network tool "${toolName}" after it was refused.`; + this.latchIssue(state, { code: 'capability_violation', message: reason }); + return { block: true, reason }; + } + return { + block: true, + reason: + `"${toolName}" is unavailable: Goal verification runs offline. ` + + 'Verify the objective from local evidence instead: read the files, search the tree, ' + + 'and inspect the repository with git. If the evidence you need is not available ' + + 'locally, judge the objective PARTIAL. Do not call web_fetch or web_search again.', + }; + } + private latchIssue(state: GoalVerifierChildRunState, issue: GoalVerifierChildRunIssue): void { state.issue ??= issue; } diff --git a/packages/local-runtime-v2/src/application/agent/goal-verifier-reminder.ts b/packages/local-runtime-v2/src/application/agent/goal-verifier-reminder.ts index 88a448b6e..7383620ea 100644 --- a/packages/local-runtime-v2/src/application/agent/goal-verifier-reminder.ts +++ b/packages/local-runtime-v2/src/application/agent/goal-verifier-reminder.ts @@ -39,6 +39,12 @@ not change Git state: no \`git add\`, \`stash\`, \`checkout\`, \`restore\`, the working tree, or refs. Inspecting with \`git status\`, \`git diff\`, and \`git log\` is expected and safe. +This run is also offline: \`web_fetch\` and \`web_search\` are not available, +and any call to them is rejected. Verify the objective from local evidence — +read the files, search the tree, and inspect the repository with git. If the +evidence you need does not exist locally, that is an evidence gap: return +PARTIAL rather than reaching for the network. + Include exactly one verdict statement in your reply, using one of these tokens: VERDICT: PASS diff --git a/packages/local-runtime-v2/src/service/turn-system/agent-host/assembly/local-turn-tool-catalog-source.ts b/packages/local-runtime-v2/src/service/turn-system/agent-host/assembly/local-turn-tool-catalog-source.ts index 853d21cd7..b3407347f 100644 --- a/packages/local-runtime-v2/src/service/turn-system/agent-host/assembly/local-turn-tool-catalog-source.ts +++ b/packages/local-runtime-v2/src/service/turn-system/agent-host/assembly/local-turn-tool-catalog-source.ts @@ -1,7 +1,11 @@ import type { RuntimeTool } from '@mavis/agent-core/tools'; import type { ModelContextAssemblyCtx } from '@mavis/agent-runtime'; import type { IModelCapabilities } from '@mavis/protocol'; -import type { ResolvedAgentCapabilities } from '@mavis/config'; +import { + GOAL_VERIFIER_OFFLINE_TOOL_NAMES, + GOAL_VERIFIER_READONLY_PROFILE, + type ResolvedAgentCapabilities, +} from '@mavis/config'; import type { LocalTurnAgentProfileFacts } from './local-turn-tool-catalog.js'; import type { AgentHostTurnCapabilityView } from './turn-capability-lifecycle.js'; @@ -33,6 +37,8 @@ export interface LocalTurnToolCatalogBuildInput { readonly userText?: string; readonly agentProfile?: LocalTurnAgentProfileFacts; readonly desktopCapabilities?: AgentHostTurnCapabilityView; + /** Tool names this Turn must not receive from any source. */ + readonly withheldToolNames?: readonly string[]; } export function createLocalTurnToolCatalogInput( @@ -42,6 +48,7 @@ export function createLocalTurnToolCatalogInput( const modelCapabilities = readModelCapabilities(context.model); const builtinCapabilities = readBuiltinCapabilities(context.agentConfig); const agentProfile = readAgentProfile(context.agentConfig); + const withheldToolNames = readWithheldToolNames(context.turnIntent); return { sessionId: context.sessionId, turnId: context.turnId, @@ -56,9 +63,25 @@ export function createLocalTurnToolCatalogInput( ...(modelCapabilities ? { modelCapabilities } : {}), ...(agentProfile ? { agentProfile } : {}), ...(desktopCapabilities ? { desktopCapabilities } : {}), + ...(withheldToolNames ? { withheldToolNames } : {}), }; } +/** + * The Goal verifier child is an ordinary `verifier` delegation, and that role + * holds `web_fetch` and `web_search`. Goal verification runs offline, so the + * child's Turn is assembled without them rather than offered tools that its + * runtime guard would then refuse. + */ +function readWithheldToolNames( + intent: ModelContextAssemblyCtx['turnIntent'], +): readonly string[] | undefined { + return intent?.kind === 'goal-verifier' && + intent.attributes?.profile === GOAL_VERIFIER_READONLY_PROFILE + ? GOAL_VERIFIER_OFFLINE_TOOL_NAMES + : undefined; +} + function readEffectivePluginSkills( agentConfig: Readonly>, ): readonly { readonly pluginName: string; readonly name: string }[] { diff --git a/packages/local-runtime-v2/src/service/turn-system/agent-host/assembly/local-turn-tool-catalog.ts b/packages/local-runtime-v2/src/service/turn-system/agent-host/assembly/local-turn-tool-catalog.ts index 4bca123ef..807918d7d 100644 --- a/packages/local-runtime-v2/src/service/turn-system/agent-host/assembly/local-turn-tool-catalog.ts +++ b/packages/local-runtime-v2/src/service/turn-system/agent-host/assembly/local-turn-tool-catalog.ts @@ -108,6 +108,8 @@ export interface BuildLocalTurnToolCatalogInput { readonly name: string; }[]; readonly userText?: string; + /** Tool names this Turn must not receive from any source (native, MCP, plugin). */ + readonly withheldToolNames?: readonly string[]; } /** The already-gated inventory shared by Turn assembly and Task capture. */ @@ -130,8 +132,12 @@ export function filterLocalTurnCapabilityInventory(input: { readonly agentProfile?: LocalTurnAgentProfileFacts; readonly desktopCapabilities?: AgentHostTurnCapabilityView; readonly hostCapabilityRegistry?: HostCapabilityResolver; + readonly withheldToolNames?: readonly string[]; }): FilteredLocalTurnCapabilityInventory { - const excludedNames = getSuppressedToolNamesForModelCapabilities(input.modelCapabilities); + const excludedNames = new Set([ + ...getSuppressedToolNamesForModelCapabilities(input.modelCapabilities), + ...(input.withheldToolNames ?? []), + ]); const profile = input.agentProfile; const selector = createLocalAgentCapabilitySelector(profile?.configSelection); const hasBrowserHostAlias = Boolean( diff --git a/packages/local-runtime-v2/src/service/turn-system/agent-host/native-production-dependencies.ts b/packages/local-runtime-v2/src/service/turn-system/agent-host/native-production-dependencies.ts index 80a2eb86a..3c5f35eb8 100644 --- a/packages/local-runtime-v2/src/service/turn-system/agent-host/native-production-dependencies.ts +++ b/packages/local-runtime-v2/src/service/turn-system/agent-host/native-production-dependencies.ts @@ -342,6 +342,7 @@ function toolCatalogBuildOptions( ...(input.desktopCapabilities ? { desktopCapabilities: input.desktopCapabilities } : {}), ...(input.modelCapabilities ? { modelCapabilities: input.modelCapabilities } : {}), ...(input.agentProfile ? { agentProfile: input.agentProfile } : {}), + ...(input.withheldToolNames ? { withheldToolNames: input.withheldToolNames } : {}), ...(options.config ? { config: options.config } : {}), ...(options.env ? { env: options.env } : {}), ...(options.emitDiagnostic ? { emitDiagnostic: options.emitDiagnostic } : {}), diff --git a/packages/local-runtime-v2/test/unit/agent/goal-verifier-readonly-guard.test.ts b/packages/local-runtime-v2/test/unit/agent/goal-verifier-readonly-guard.test.ts new file mode 100644 index 000000000..91c24b5fe --- /dev/null +++ b/packages/local-runtime-v2/test/unit/agent/goal-verifier-readonly-guard.test.ts @@ -0,0 +1,343 @@ +import type { RuntimeTool } from '@mavis/agent-core/tools'; +import type { McpToolEntry } from '@mavis/agent-tools'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { GOAL_VERIFIER_READONLY_PROFILE, resolveAgentCapabilities } from '@mavis/config'; +import { Type } from '@sinclair/typebox'; +import { describe, expect, it } from 'vitest'; + +import { GoalVerifierChildCoordinator } from '../../../src/application/agent/goal-subagent-coordinator.js'; +import { renderGoalVerifierReminder } from '../../../src/application/agent/goal-verifier-reminder.js'; +import { BuiltinAgentCatalog } from '../../../src/service/agent/builtin/catalog.js'; +import { createLocalTurnToolCatalogInput } from '../../../src/service/turn-system/agent-host/assembly/local-turn-tool-catalog-source.js'; +import { buildLocalTurnToolCatalog } from '../../../src/service/turn-system/agent-host/assembly/local-turn-tool-catalog.js'; + +const HOST_PROVIDER = 'worker-provider'; +const VIOLATION = 'GOAL_VERIFIER_CAPABILITY_VIOLATION'; + +function register( + coordinator: GoalVerifierChildCoordinator, + options: { readonly maxTurns?: number } = {}, +): void { + coordinator.register({ + runId: 'run-1', + workerRouteProvider: HOST_PROVIDER, + hostContext: { settlement: {} } as never, + ...options, + }); +} + +/** Minimal `before_tool_call` context: the guard only reads the tool name. */ +function toolCall(name: string): never { + return { toolCall: { name } } as never; +} + +/** Starts the next child turn and asserts the host let it run. */ +function nextTurn(coordinator: GoalVerifierChildCoordinator): void { + expect(coordinator.beforeLlm('run-1', HOST_PROVIDER)).toBeUndefined(); +} + +describe('Goal verifier child network tool guard', () => { + it('refuses a network tool once without aborting the run', () => { + const coordinator = new GoalVerifierChildCoordinator(); + register(coordinator); + nextTurn(coordinator); + + const refused = coordinator.beforeTool('run-1', toolCall('web_fetch')); + + expect(refused?.block).toBe(true); + expect(refused?.reason).not.toContain(VIOLATION); + expect(refused?.reason).toContain('offline'); + expect(refused?.reason).toContain('local evidence'); + expect(refused?.reason).toContain('PARTIAL'); + // Before the fix the refusal latched a violation and this turn was aborted, + // which surfaced as `paused(verifier_capability)`. + nextTurn(coordinator); + expect(coordinator.release('run-1')?.issue).toBeUndefined(); + }); + + it('counts parallel network calls from one model response as one turn', () => { + const coordinator = new GoalVerifierChildCoordinator(); + register(coordinator); + nextTurn(coordinator); + + for (const name of ['web_fetch', 'web_search', 'web_fetch', 'web_search']) { + const refused = coordinator.beforeTool('run-1', toolCall(name)); + expect(refused?.block).toBe(true); + expect(refused?.reason).not.toContain(VIOLATION); + } + + nextTurn(coordinator); + expect(coordinator.release('run-1')?.issue).toBeUndefined(); + }); + + it('latches a capability violation when network tools are called in consecutive turns', () => { + const coordinator = new GoalVerifierChildCoordinator(); + register(coordinator); + nextTurn(coordinator); + coordinator.beforeTool('run-1', toolCall('web_fetch')); + nextTurn(coordinator); + + const latched = coordinator.beforeTool('run-1', toolCall('web_search')); + + expect(latched).toEqual({ block: true, reason: expect.stringContaining(VIOLATION) }); + expect(coordinator.beforeLlm('run-1', HOST_PROVIDER)).toEqual({ + type: 'abort', + reason: expect.stringContaining(VIOLATION), + }); + expect(coordinator.release('run-1')?.issue?.code).toBe('capability_violation'); + }); + + it('bounds network calls that alternate with local work', () => { + const coordinator = new GoalVerifierChildCoordinator(); + register(coordinator); + const refusalPerTurn: (string | undefined)[] = []; + for (let turn = 0; turn < 5; turn += 1) { + nextTurn(coordinator); + expect(coordinator.beforeTool('run-1', toolCall('read'))).toBeUndefined(); + if (turn % 2 === 0) { + refusalPerTurn.push(coordinator.beforeTool('run-1', toolCall('web_fetch'))?.reason); + } + } + + expect(refusalPerTurn[0]).not.toContain(VIOLATION); + expect(refusalPerTurn[1]).not.toContain(VIOLATION); + expect(refusalPerTurn[2]).toContain(VIOLATION); + expect(coordinator.release('run-1')?.issue?.code).toBe('capability_violation'); + }); + + it.each(['write', 'edit', 'todowrite', 'task', 'memory', 'desktop_click'])( + 'still stops the run on the first call to non-network tool %s', + (name) => { + const coordinator = new GoalVerifierChildCoordinator(); + register(coordinator); + nextTurn(coordinator); + + expect(coordinator.beforeTool('run-1', toolCall(name))).toEqual({ + block: true, + reason: expect.stringContaining(VIOLATION), + }); + expect(coordinator.beforeLlm('run-1', HOST_PROVIDER)).toEqual({ + type: 'abort', + reason: expect.stringContaining(VIOLATION), + }); + expect(coordinator.release('run-1')?.issue?.code).toBe('capability_violation'); + }, + ); + + it('leaves the read-only verifier tools untouched', () => { + const coordinator = new GoalVerifierChildCoordinator(); + register(coordinator); + for (let turn = 0; turn < 3; turn += 1) { + nextTurn(coordinator); + for (const name of ['read', 'grep', 'glob', 'bash', 'task_query', 'task_output']) { + expect(coordinator.beforeTool('run-1', toolCall(name))).toBeUndefined(); + } + } + expect(coordinator.release('run-1')?.issue).toBeUndefined(); + }); + + it('still refuses every tool for an unknown run', () => { + const coordinator = new GoalVerifierChildCoordinator(); + + expect(coordinator.beforeTool('missing-run', toolCall('web_fetch'))).toEqual({ + block: true, + reason: expect.stringContaining('no longer active'), + }); + }); + + it('still aborts a route mismatch immediately', () => { + const coordinator = new GoalVerifierChildCoordinator(); + register(coordinator); + + expect(coordinator.beforeLlm('run-1', 'some-other-provider')).toEqual({ + type: 'abort', + reason: expect.stringContaining('settled worker data route'), + }); + expect(coordinator.release('run-1')?.issue?.code).toBe('route_unavailable'); + }); + + it('still aborts at the host turn cap', () => { + const coordinator = new GoalVerifierChildCoordinator(); + register(coordinator, { maxTurns: 1 }); + nextTurn(coordinator); + + expect(coordinator.beforeLlm('run-1', HOST_PROVIDER)).toEqual({ + type: 'abort', + reason: expect.stringContaining('execution cap'), + }); + expect(coordinator.release('run-1')?.issue?.code).toBe('child_budget_exhausted'); + }); + + it('tells the child up front that verification is offline', () => { + const reminder = renderGoalVerifierReminder({ + completionProposal: { turnId: 'turn-1' }, + settlement: { durableStatusAtDispatch: 'active', transitionOnMet: 'complete' }, + } as never); + + expect(reminder).toContain('This run is also offline'); + expect(reminder).toContain('`web_fetch` and `web_search` are not available'); + }); +}); + +describe('Goal verifier child tool catalog', () => { + const VERIFIER_INTENT = { + kind: 'goal-verifier', + attributes: { runId: 'run-1', profile: GOAL_VERIFIER_READONLY_PROFILE }, + }; + const TOOL_CONTEXT = { sessionId: 'child-session', turnId: 'child-turn' }; + + function tool(name: string): RuntimeTool { + return { + def: { name, description: `Synthetic ${name} tool`, schema: Type.Object({}) }, + impl: { + execute: async () => ({ + tool_name: name, + text: 'ok', + content: [{ type: 'text' as const, text: 'ok' }], + }), + }, + }; + } + + /** The builtin Matrix MCP server contributes `web_search` under the same name. */ + function matrixWebSearch(): McpToolEntry { + return { tool: tool('web_search'), source: 'builtin-matrix' } as never; + } + + function catalogInput(turnIntent?: { kind: string; attributes?: Record }) { + return createLocalTurnToolCatalogInput({ + sessionId: 'child-session', + turnId: 'child-turn', + agentName: 'verifier', + workspaceDir: '/workspace', + agentConfig: {}, + model: { provider: HOST_PROVIDER, model_id: 'offline', context_window: 128_000 }, + history: [] as never, + ...(turnIntent ? { turnIntent } : {}), + }); + } + + function buildCatalog(input: { + readonly nativeTools: readonly string[]; + readonly mcpEntries: readonly McpToolEntry[]; + readonly withheldToolNames?: readonly string[]; + readonly deferMcp?: boolean; + }) { + return buildLocalTurnToolCatalog({ + sessionId: 'child-session', + llmModel: { provider: HOST_PROVIDER, id: 'offline', contextWindow: 128_000 }, + sources: { + nativeTools: input.nativeTools.map(tool), + mcpEntries: input.mcpEntries, + threadGoalTools: [], + cuRuntimeAvailable: false, + }, + env: {}, + ...(input.deferMcp ? { config: { modelWhitelist: ['*'], thresholdPct: 0 } } : {}), + ...(input.withheldToolNames ? { withheldToolNames: input.withheldToolNames } : {}), + }); + } + + function toolNames(catalog: ReturnType): string[] { + return catalog.tools.map((entry) => entry.def.name); + } + + it('derives the offline tool names only for the readonly Goal verifier intent', () => { + expect(catalogInput(VERIFIER_INTENT).withheldToolNames).toEqual(['web_fetch', 'web_search']); + expect(catalogInput().withheldToolNames).toBeUndefined(); + expect( + catalogInput({ kind: 'goal-verifier', attributes: { runId: 'run-1', profile: 'other' } }) + .withheldToolNames, + ).toBeUndefined(); + }); + + it('withholds native network tools from a Goal verifier child turn', () => { + const nativeTools = ['read', 'grep', 'glob', 'bash', 'web_fetch', 'web_search']; + const withheld = toolNames( + buildCatalog({ + nativeTools, + mcpEntries: [], + withheldToolNames: catalogInput(VERIFIER_INTENT).withheldToolNames, + }), + ); + + expect(withheld).toEqual(expect.arrayContaining(['read', 'grep', 'glob', 'bash'])); + expect(withheld).not.toContain('web_fetch'); + expect(withheld).not.toContain('web_search'); + expect(toolNames(buildCatalog({ nativeTools, mcpEntries: [] }))).toEqual( + expect.arrayContaining(['web_fetch', 'web_search']), + ); + }); + + it('withholds the builtin Matrix MCP web_search when it is offered inline', () => { + const withheld = buildCatalog({ + nativeTools: ['read'], + mcpEntries: [matrixWebSearch()], + withheldToolNames: catalogInput(VERIFIER_INTENT).withheldToolNames, + }); + const offered = buildCatalog({ nativeTools: ['read'], mcpEntries: [matrixWebSearch()] }); + + expect(toolNames(offered)).toContain('web_search'); + expect(toolNames(withheld)).toEqual(['read']); + }); + + it('keeps the builtin Matrix MCP web_search out of a deferred MCP catalog', async () => { + const notes = { + tool: tool('mcp__notes__read'), + source: 'configured', + serverName: 'notes-server', + } as never as McpToolEntry; + const deferredCatalog = (withheldToolNames?: readonly string[]) => + buildCatalog({ + nativeTools: ['read'], + mcpEntries: [matrixWebSearch(), notes], + deferMcp: true, + ...(withheldToolNames ? { withheldToolNames } : {}), + }); + + // Ordinary turns keep Matrix web_search while other MCP tools are deferred. + const offered = toolNames(deferredCatalog()); + expect(offered).toEqual(expect.arrayContaining(['web_search', 'tool_search', 'mcp_invoke'])); + expect(offered).not.toContain('mcp__notes__read'); + + const withheld = deferredCatalog(catalogInput(VERIFIER_INTENT).withheldToolNames); + expect(toolNames(withheld)).toEqual(expect.arrayContaining(['tool_search', 'mcp_invoke'])); + expect(toolNames(withheld)).not.toContain('web_search'); + // Nor can the deferred path reach it by name. + const invoke = withheld.tools.find((entry) => entry.def.name === 'mcp_invoke'); + if (!invoke) throw new Error('mcp_invoke is missing from the deferred catalog.'); + const result = await invoke.impl + .execute(TOOL_CONTEXT, { tool_name: 'web_search', arguments: {} }) + .catch((error: unknown) => error); + expect(result).not.toMatchObject({ text: 'ok' }); + await expect( + invoke.impl.execute(TOOL_CONTEXT, { tool_name: 'mcp__notes__read', arguments: {} }), + ).resolves.toMatchObject({ text: 'ok' }); + }); + + it.each([{ promptProfile: 'tui' }, { promptProfile: 'desktop' }])( + 'renders no web-search guidance into the verifier task-child prompt (%o)', + async (profile) => { + const assetsDir = resolve(dirname(fileURLToPath(import.meta.url)), '../../../assets/agents'); + const rendered = await new BuiltinAgentCatalog({ assetsDir }).render({ + agentName: 'verifier', + surface: 'task-child', + appMode: 'work', + locale: 'en', + promptChannel: 'online', + // The verifier role's own grant: webSearch stays on at the role level. + capabilities: resolveAgentCapabilities({ features: { webSearch: true } }), + ...profile, + } as never); + const prompt = [rendered.persona ?? '', rendered.corePrompt, rendered.surfacePrompt].join( + '\n', + ); + + expect(prompt).not.toContain('Factual Freshness And Search'); + expect(prompt).not.toContain('web_search'); + expect(prompt).not.toContain('web_fetch'); + }, + ); +}); diff --git a/release/public-source.json b/release/public-source.json index a74c8fc27..7b39a36f1 100644 --- a/release/public-source.json +++ b/release/public-source.json @@ -1525,6 +1525,7 @@ "packages/local-runtime-v2/test/integration/session-system-queue-repository.integration.test.ts", "packages/local-runtime-v2/test/unit/agent/agent-import.test.ts", "packages/local-runtime-v2/test/unit/agent/builtin/catalog.test.ts", + "packages/local-runtime-v2/test/unit/agent/goal-verifier-readonly-guard.test.ts", "packages/local-runtime-v2/test/unit/agent/storage/agent.repository.test.ts", "packages/local-runtime-v2/test/unit/agent/storage/canonical-agent-config.test.ts", "packages/local-runtime-v2/test/unit/compat/v1/runtime.test.ts", diff --git a/test/vitest-suites.json b/test/vitest-suites.json index 80c0ee07b..cb18182dc 100644 --- a/test/vitest-suites.json +++ b/test/vitest-suites.json @@ -123,6 +123,7 @@ "packages/local-runtime-v2/test/unit/agent/builtin/catalog.test.ts", "packages/local-runtime-v2/test/unit/agent/storage/agent.repository.test.ts", "packages/local-runtime-v2/test/unit/agent/storage/canonical-agent-config.test.ts", + "packages/local-runtime-v2/test/unit/agent/goal-verifier-readonly-guard.test.ts", "packages/local-runtime-v2/test/unit/compat/v1/runtime.test.ts", "packages/tui/test/unit/runtime-product-access.test.ts", "packages/tui/test/unit/tui-app.test.ts",