From 5a7b4c96cc9b47fb7130316296f5e8f5174cc49f Mon Sep 17 00:00:00 2001 From: miguel Date: Mon, 7 Sep 2026 16:49:21 -0700 Subject: [PATCH 1/5] Add Gemini native computer use with model-visible evidence --- .github/workflows/ci.yml | 1 + packages/evals/README.md | 8 +- packages/evals/core/contracts/tool.ts | 1 + packages/evals/core/tools/registry.ts | 5 + packages/evals/core/tools/stagehand_facade.ts | 5 + packages/evals/docs/harness-contract.md | 60 ++++ packages/evals/framework/benchHarness.ts | 11 + packages/evals/framework/geminiCuaRunner.ts | 128 +++++++ .../evals/framework/geminiCuaToolAdapter.ts | 82 +++++ .../framework/harnesses/geminiCuaAdapter.ts | 71 ++++ packages/evals/package.json | 5 +- .../tests/framework/benchHarness.test.ts | 3 +- .../tests/framework/cuaToolAdapter.test.ts | 10 +- .../tests/framework/geminiCuaAdapter.test.ts | 148 ++++++++ .../tests/framework/geminiCuaMount.test.ts | 91 +++++ .../tests/framework/geminiCuaRunner.test.ts | 83 +++++ .../gemini-cua-sdk/ARCHITECTURE.md | 15 + .../integrations/gemini-cua-sdk/package.json | 36 ++ .../gemini-cua-sdk/src/executor.ts | 285 +++++++++++++++ .../integrations/gemini-cua-sdk/src/index.ts | 3 + .../gemini-cua-sdk/src/session.ts | 337 ++++++++++++++++++ .../gemini-cua-sdk/src/transcript.ts | 25 ++ .../gemini-cua-sdk/tests/executor.test.ts | 130 +++++++ .../gemini-cua-sdk/tests/session.test.ts | 274 ++++++++++++++ .../integrations/gemini-cua-sdk/tsconfig.json | 5 + .../gemini-cua-sdk/tsdown.config.ts | 10 + pnpm-lock.yaml | 31 ++ pnpm-workspace.yaml | 1 + turbo.json | 18 + vitest.config.ts | 1 + 30 files changed, 1878 insertions(+), 5 deletions(-) create mode 100644 packages/evals/docs/harness-contract.md create mode 100644 packages/evals/framework/geminiCuaRunner.ts create mode 100644 packages/evals/framework/geminiCuaToolAdapter.ts create mode 100644 packages/evals/framework/harnesses/geminiCuaAdapter.ts create mode 100644 packages/evals/tests/framework/geminiCuaAdapter.test.ts create mode 100644 packages/evals/tests/framework/geminiCuaMount.test.ts create mode 100644 packages/evals/tests/framework/geminiCuaRunner.test.ts create mode 100644 packages/integrations/gemini-cua-sdk/ARCHITECTURE.md create mode 100644 packages/integrations/gemini-cua-sdk/package.json create mode 100644 packages/integrations/gemini-cua-sdk/src/executor.ts create mode 100644 packages/integrations/gemini-cua-sdk/src/index.ts create mode 100644 packages/integrations/gemini-cua-sdk/src/session.ts create mode 100644 packages/integrations/gemini-cua-sdk/src/transcript.ts create mode 100644 packages/integrations/gemini-cua-sdk/tests/executor.test.ts create mode 100644 packages/integrations/gemini-cua-sdk/tests/session.test.ts create mode 100644 packages/integrations/gemini-cua-sdk/tsconfig.json create mode 100644 packages/integrations/gemini-cua-sdk/tsdown.config.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 199450cdc..4ae1eb555 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -261,6 +261,7 @@ jobs: packages/integrations/core/dist/** packages/integrations/claude-agent-sdk/dist/** packages/integrations/claude-cua-sdk/dist/** + packages/integrations/gemini-cua-sdk/dist/** packages/integrations/codex-sdk/dist/** packages/integrations/mastra-sdk/dist/** packages/integrations/pi-sdk/dist/** diff --git a/packages/evals/README.md b/packages/evals/README.md index 9c7b69de7..038bd6356 100644 --- a/packages/evals/README.md +++ b/packages/evals/README.md @@ -1,6 +1,6 @@ # Stagehand Evals -Agent benchmarks for Stagehand — `act`, `extract`, `observe`, `agent`, plus dataset-backed suites (WebVoyager, OnlineMind2Web, WebTailBench, Odysseys). +Agent benchmarks for Stagehand — `act`, `extract`, `observe`, `agent`, plus dataset-backed suites (WebVoyager, OnlineMind2Web, WebTailBench, Odysseys, HardBench). Driven by an interactive TUI (`evals`) or single-shot CLI (`evals run …`). Tasks are auto-discovered from `tasks/bench//` — no registration step. @@ -84,6 +84,12 @@ A live run paints an in-place progress table, then prints a final summary with a ![Live bench run](./assets/readme/run.gif) +## Shared harness behavior + +See [the harness contract](docs/harness-contract.md) for tool surfaces, prompt policy, +budget units, session diagnostics, verification, and usage accounting. +See [HardBench](datasets/hardbenchmark/MANIFEST.md) for corpus selection and rubric v1.2. + ## Adding a bench task ```bash diff --git a/packages/evals/core/contracts/tool.ts b/packages/evals/core/contracts/tool.ts index 053b849a4..7ea758086 100644 --- a/packages/evals/core/contracts/tool.ts +++ b/packages/evals/core/contracts/tool.ts @@ -11,6 +11,7 @@ export type ToolSurface = | "cdp_code" | "playwright_mcp" | "chrome_devtools_mcp" + | "google_computer_use" | "anthropic_browser_toolset" | "stagehand_facade" | "stagehand_facade_legacy" diff --git a/packages/evals/core/tools/registry.ts b/packages/evals/core/tools/registry.ts index dc32fdd79..621344bbf 100644 --- a/packages/evals/core/tools/registry.ts +++ b/packages/evals/core/tools/registry.ts @@ -9,11 +9,13 @@ import { StagehandFacadeTool, StagehandFacadeLegacyTool, AnthropicBrowserToolsetTool, + GoogleComputerUseTool, } from "./stagehand_facade.js"; import { UnderstudyCodeTool } from "./understudy_code.js"; /** Surfaces that exist only as an agent MCP mount; they have no runner-driven CoreSession (activePage() throws). */ export const AGENT_MOUNT_ONLY_TOOL_SURFACES: ReadonlySet = new Set([ + "google_computer_use", "anthropic_browser_toolset", "stagehand_facade", "stagehand_facade_legacy", @@ -33,6 +35,7 @@ export function listCoreTools(): ToolSurface[] { "chrome_devtools_mcp", // Listed here as part of the full enumeration, but agent-mount-only: // core-tier selection must use listCoreRunnableTools, which filters it. + "google_computer_use", "anthropic_browser_toolset", "stagehand_facade", "stagehand_facade_legacy", @@ -59,6 +62,8 @@ export function getCoreTool(toolSurface: ToolSurface): CoreTool { return new PlaywrightMcpTool(); case "chrome_devtools_mcp": return new ChromeDevtoolsMcpTool(); + case "google_computer_use": + return new GoogleComputerUseTool(); case "anthropic_browser_toolset": return new AnthropicBrowserToolsetTool(); case "stagehand_facade": diff --git a/packages/evals/core/tools/stagehand_facade.ts b/packages/evals/core/tools/stagehand_facade.ts index d42c88aea..4452dd3a0 100644 --- a/packages/evals/core/tools/stagehand_facade.ts +++ b/packages/evals/core/tools/stagehand_facade.ts @@ -318,3 +318,8 @@ export class StagehandFacadeLegacyTool extends StagehandFacadeTool { export class AnthropicBrowserToolsetTool extends StagehandFacadeTool { override readonly id: ToolSurface = "anthropic_browser_toolset"; } + +/** Native Gemini actions execute on the same facade-owned browser. */ +export class GoogleComputerUseTool extends StagehandFacadeTool { + override readonly id: ToolSurface = "google_computer_use"; +} diff --git a/packages/evals/docs/harness-contract.md b/packages/evals/docs/harness-contract.md new file mode 100644 index 000000000..70f9e7380 --- /dev/null +++ b/packages/evals/docs/harness-contract.md @@ -0,0 +1,60 @@ +# Shared eval harness contract + +External harnesses use the runner-owned tool mount. The default is `stagehand_facade` +where supported; `stagehand_facade_legacy` remains explicitly selectable. Provider +adapters translate their protocols to that shared runtime. They do not start a +second browser or implement a separate Playwright facade. + +## Prompt and execution configuration + +The eval runner owns the no-clarification policy in `evalSystemPrompt.ts`. It is +injected once through an additive system/developer channel when available; +otherwise it is included once in the task prompt. Harness stock prompts remain +in place. A caller-supplied Codex client and remote Eve use the task-prefix path. +The Cursor SDK's replacement system-prompt option is intentionally unused. + +Budget precedence is the harness-specific environment variable, then +`AGENT_EVAL_MAX_STEPS`, then the dataset default (HardBench: 100), then the +historical harness default. Only positive safe integers are accepted. +`harnessConfiguration` records the effective budget, unit, policy channel/version, +and requested reasoning settings where supported. Equal budget numbers do not +mean equal work: + +| Harness | Counted unit | +| --------------------------------------- | --------------------- | +| Codex, Cursor, DeepAgents | Tool calls | +| Eve | Successful tool calls | +| Mastra | Model steps | +| fx | Agent steps | +| Claude Code, Pi, Claude CUA, Gemini CUA | Turns | + +These are execution limits, not comparable measures of model efficiency. + +## Session and result records + +The mounted surface supplies session identity and evidence. Session ownership, +first terminal browser loss, and cleanup live in the shared runtime. Capture +recovery permits two consecutive capture deadlines; the third ends the session. +A successful capture resets the counter. Timed-out work cannot replace newer +snapshot IDs, and failed actions are not replayed. CDP heartbeat and sanitized +`CDP_DROP` diagnostics are available; automatic reconnect is not implemented. + +`harnessStatus` and `terminationReason` describe execution. The verifier determines +completion from evidence, including work completed before a late disconnect. +When verification fails, `_success` is false, `verifierError` records the failure, +and the agent's report is preserved separately. Captured trajectories and raw +judge uncertainty are retained when persistence is enabled. See +[verification gates](verifier-gates.md) for scoring and audit fields. + +## Usage and cost + +Raw token usage is normalized according to the SDK's cache conventions. +Observed zero is distinct from missing telemetry; missing usage and unknown +subscription bills do not become zero-dollar runs. Reported bills take precedence. +Direct-provider estimates use the dated checked-in price map and include +`cost_source: computed` and `cost_pricing` (date, matched model, source). +These are list-price estimates, not invoice reconciliation or current-price claims. + +Cursor runs use the SDK through `--harness cursor` and record implementation/version +provenance. Historical CLI `cursor` and `cursor_sdk` records remain readable; +missing provenance is not retroactively interpreted as an SDK run. diff --git a/packages/evals/framework/benchHarness.ts b/packages/evals/framework/benchHarness.ts index 7d0ed6865..94a4f46b3 100644 --- a/packages/evals/framework/benchHarness.ts +++ b/packages/evals/framework/benchHarness.ts @@ -1,3 +1,5 @@ +import { runGeminiCuaAgent, GEMINI_CUA_DEFAULT_MODELS } from "./geminiCuaRunner.js"; +import { GEMINI_CUA_TOOL_SURFACES, prepareGeminiCuaToolAdapter } from "./geminiCuaToolAdapter.js"; import { runClaudeCuaAgent, CLAUDE_CUA_DEFAULT_MODELS } from "./claudeCuaRunner.js"; import { CLAUDE_CUA_TOOL_SURFACES, prepareClaudeCuaToolAdapter } from "./claudeCuaToolAdapter.js"; import { V3, normalizeRubric, type AvailableModel, type TaskSpec } from "stagehand-v3"; @@ -376,6 +378,14 @@ export const claudeCuaHarness = defineExternalHarness({ runAgent: runClaudeCuaAgent, }); +export const geminiCuaHarness = defineExternalHarness({ + harness: "gemini_cua", + supportedToolSurfaces: GEMINI_CUA_TOOL_SURFACES, + defaultModels: GEMINI_CUA_DEFAULT_MODELS, + prepareToolAdapter: prepareGeminiCuaToolAdapter, + runAgent: runGeminiCuaAgent, +}); + const harnessRegistry = new Map([ ["stagehand", stagehandHarness], ["claude_code", claudeCodeHarness], @@ -387,6 +397,7 @@ const harnessRegistry = new Map([ ["fx", fxHarness], ["cursor", cursorHarness], ["claude_cua", claudeCuaHarness], + ["gemini_cua", geminiCuaHarness], ]); export function registerBenchHarness(harness: BenchHarness): () => void { diff --git a/packages/evals/framework/geminiCuaRunner.ts b/packages/evals/framework/geminiCuaRunner.ts new file mode 100644 index 000000000..9dec24feb --- /dev/null +++ b/packages/evals/framework/geminiCuaRunner.ts @@ -0,0 +1,128 @@ +import { + buildGeminiCuaTranscript, + runGeminiCuaSession, + type GeminiGenerateClient, +} from "@browserbasehq/stagehand-integrations-gemini-cua-sdk"; +import type { AvailableModel } from "stagehand-v3"; +import { EvalsError } from "../errors.js"; +import type { EvalLogger } from "../logger.js"; +import type { PreparedGeminiCuaToolAdapter } from "./geminiCuaToolAdapter.js"; +import type { ExternalHarnessTaskPlan } from "./externalHarnessPlan.js"; +import { + buildExternalHarnessPrompt, + metricValue, + runExternalHarnessTask, +} from "./harnesses/externalRunner.js"; +import { resolveStepBudget } from "./stepBudget.js"; +import type { TaskResult } from "./types.js"; +import type { ExternalHarnessVerifierConfig } from "./verifierAdapter.js"; +import { geminiCuaAdapter } from "./harnesses/geminiCuaAdapter.js"; +export const GEMINI_CUA_DEFAULT_MODELS: AvailableModel[] = [ + "google/gemini-3.8-flash" as AvailableModel, +]; +export const GEMINI_CUA_SYSTEM_PROMPT = + "You are being evaluated on a browser task. Complete it with computer use only and finish by printing the requested EVAL_RESULT line."; +export interface GeminiCuaRunnerInput { + plan: ExternalHarnessTaskPlan; + model: AvailableModel; + logger: EvalLogger; + toolAdapter: PreparedGeminiCuaToolAdapter; + signal?: AbortSignal; + verifier?: ExternalHarnessVerifierConfig; + client?: GeminiGenerateClient; +} +export function buildGeminiCuaPrompt( + plan: ExternalHarnessTaskPlan, + toolInstructions?: string, +): string { + return buildExternalHarnessPrompt({ plan, toolInstructions, resultContract: "marker" }); +} +export function assertGeminiCuaModel(model: string): void { + const bare = model.includes("/") ? model.slice(model.indexOf("/") + 1) : model; + const provider = model.includes("/") ? model.slice(0, model.indexOf("/")) : undefined; + if ((provider !== undefined && provider !== "google") || !bare.startsWith("gemini-")) + throw new EvalsError(`gemini_cua runs Google Gemini models only (received "${model}").`); +} +export async function runGeminiCuaAgent(input: GeminiCuaRunnerInput): Promise { + assertGeminiCuaModel(input.model); + const maxTurns = resolveStepBudget({ + harnessEnvKey: "EVAL_GEMINI_CUA_MAX_STEPS", + dataset: input.plan.dataset, + harnessDefault: 50, + }); + return runExternalHarnessTask({ + harness: "gemini_cua", + plan: input.plan, + model: input.model, + logger: input.logger, + toolAdapter: input.toolAdapter, + verifier: input.verifier, + resultContract: "marker", + fallbackErrorMessage: "gemini_cua did not report success", + stepBudget: maxTurns, + stepBudgetUnit: "turns", + systemPromptMode: "native", + runSession: async (prompt, systemPrompt) => { + const result = await runGeminiCuaSession({ + prompt, + model: input.model, + logger: input.logger, + maxTurns, + tools: input.toolAdapter.executor, + facade: input.toolAdapter.facade, + systemPrompt: `${systemPrompt}\n\n${GEMINI_CUA_SYSTEM_PROMPT}`, + signal: input.signal, + client: input.client, + }); + return { + raw: result, + resultText: result.finalMessage, + transcriptText: buildGeminiCuaTranscript(result.events), + iterationError: result.iterationError, + status: result.status, + stopReason: + result.status === "sdk_error" + ? result.stopReason || stringifyError(result.iterationError) || undefined + : result.stopReason, + usage: { + reported: result.usageReported, + inputTokens: result.tokenUsage.input, + outputTokens: result.tokenUsage.output, + cachedInputTokens: result.tokenUsage.cached_input, + reasoningOutputTokens: result.tokenUsage.reasoning, + totalTokens: result.tokenUsage.total, + }, + metrics: { + gemini_cua_turns: metricValue(result.turns), + gemini_cua_tool_calls: metricValue(result.toolCalls), + }, + }; + }, + toTrajectory: ({ raw, parsed, finalObservation, status }, taskSpec) => + geminiCuaAdapter.fromHarnessResult( + { + events: raw.events, + finalAnswer: parsed.finalAnswer ?? raw.finalMessage, + status, + ...(finalObservation && { finalObservation }), + usage: { + input_tokens: raw.tokenUsage.input, + output_tokens: raw.tokenUsage.output, + cached_input_tokens: raw.tokenUsage.cached_input, + }, + }, + taskSpec, + ), + }); +} + +function stringifyError(value: unknown): string { + if (!value) return ""; + if (value instanceof Error) return value.message; + if (typeof value === "string") return value; + try { + return JSON.stringify(value) ?? String(value); + } catch { + return String(value); + } +} diff --git a/packages/evals/framework/geminiCuaToolAdapter.ts b/packages/evals/framework/geminiCuaToolAdapter.ts new file mode 100644 index 000000000..9d69a684c --- /dev/null +++ b/packages/evals/framework/geminiCuaToolAdapter.ts @@ -0,0 +1,82 @@ +import { + GeminiCuaExecutor, + type CuaFacadeTools, + type GeminiToolExecutor, +} from "@browserbasehq/stagehand-integrations-gemini-cua-sdk"; +import type { ProbeEvidence } from "stagehand-v3"; +import type { BrowserSessionLoss, StartupProfile, ToolSurface } from "../core/contracts/tool.js"; +import { bridgeCuaFacadeTools, cuaCleanup } from "./cuaToolAdapter.js"; +import { EvalsError } from "../errors.js"; +import type { EvalLogger } from "../logger.js"; +import type { BrowserSessionInfo } from "./browserSession.js"; +import type { ExternalHarnessTaskPlan } from "./externalHarnessPlan.js"; +import { startAgentToolRuntime } from "./agentToolRuntime.js"; +import { resolveStartupProfile, resolveToolSurface } from "./harnesses/toolSurfaceResolution.js"; + +export const GEMINI_CUA_TOOL_INSTRUCTIONS = + "Browser tool surface: Gemini Computer Use.\nUse the computer-use actions supplied by the API to control the existing browser. Action coordinates use the normalized 0–999 range on a 1288×711 viewport. Each action response includes the current URL and a fresh screenshot when available. Use the supplied action schemas for their exact arguments."; + +export const GEMINI_CUA_TOOL_SURFACES: ToolSurface[] = ["google_computer_use"]; +export interface PreparedGeminiCuaToolAdapter { + toolSurface: ToolSurface; + startupProfile: StartupProfile; + promptInstructions: string; + browserSession: BrowserSessionInfo; + executor: GeminiToolExecutor; + facade: CuaFacadeTools; + captureEvidence?: () => Promise; + browserSessionLoss?: () => BrowserSessionLoss | undefined; + observedToolMatcher: (name: string) => boolean; + cleanup: () => Promise; +} + +export async function prepareGeminiCuaToolAdapter(input: { + toolSurface?: ToolSurface; + startupProfile?: StartupProfile; + environment: "LOCAL" | "BROWSERBASE"; + plan: ExternalHarnessTaskPlan; + logger: EvalLogger; +}): Promise { + const toolSurface = resolveToolSurface( + { harness: "gemini_cua", supportedToolSurfaces: GEMINI_CUA_TOOL_SURFACES }, + input.toolSurface, + ); + if (!toolSurface) throw new EvalsError("gemini_cua harness requires a tool surface."); + const startupProfile = resolveStartupProfile( + toolSurface, + input.environment, + input.startupProfile, + ); + const runtime = await startAgentToolRuntime({ + toolSurface, + startupProfile, + environment: input.environment, + logger: input.logger, + }); + const cleanup = cuaCleanup(runtime.cleanup); + try { + const callTool = runtime.running.callTool; + const mount = runtime.running.agentMount; + if (!callTool || !mount) + throw new EvalsError(`Tool surface "${toolSurface}" does not expose runner-side tool calls.`); + const facade = bridgeCuaFacadeTools(callTool); + const executor = new GeminiCuaExecutor(facade, input.logger); + return { + toolSurface, + startupProfile, + promptInstructions: GEMINI_CUA_TOOL_INSTRUCTIONS, + browserSession: runtime.browserSession, + executor, + facade, + ...(runtime.running.captureEvidence && { captureEvidence: runtime.running.captureEvidence }), + ...(runtime.running.browserSessionLoss && { + browserSessionLoss: runtime.running.browserSessionLoss, + }), + observedToolMatcher: () => true, + cleanup, + }; + } catch (error) { + await cleanup(); + throw error; + } +} diff --git a/packages/evals/framework/harnesses/geminiCuaAdapter.ts b/packages/evals/framework/harnesses/geminiCuaAdapter.ts new file mode 100644 index 000000000..61be2a979 --- /dev/null +++ b/packages/evals/framework/harnesses/geminiCuaAdapter.ts @@ -0,0 +1,71 @@ +import type { ProbeEvidence, TaskSpec, Trajectory } from "stagehand-v3"; +import type { GeminiCuaSessionEvent } from "@browserbasehq/stagehand-integrations-gemini-cua-sdk"; +import type { StepObservation } from "../observationRecorder.js"; +import { + buildTrajectory, + type NormalizedToolCall, + type TrajectoryAdapter, +} from "./trajectoryAdapter.js"; + +export interface GeminiCuaRunResult { + events: GeminiCuaSessionEvent[]; + finalAnswer?: string; + status?: Trajectory["status"]; + usage?: Partial; + finalObservation?: ProbeEvidence; + stepObservations?: StepObservation[]; + stepObservationsByToolUse?: Map; +} + +export class GeminiCuaTrajectoryAdapter implements TrajectoryAdapter { + fromHarnessResult(result: GeminiCuaRunResult, taskSpec: TaskSpec): Trajectory { + const toolCalls: NormalizedToolCall[] = []; + const byId = new Map(); + let pendingReasoning = ""; + let finalAnswer = result.finalAnswer; + + for (const event of result.events) { + if (event.type === "assistant") { + pendingReasoning = event.text; + if (!result.finalAnswer && event.text) finalAnswer = event.text; + } else if (event.type === "tool_use") { + const call: NormalizedToolCall = { + name: event.name, + args: event.input, + result: undefined, + ok: true, + reasoning: pendingReasoning.trim() || undefined, + probeEvidence: + result.stepObservationsByToolUse?.get(event.id) ?? + result.stepObservations?.find( + (observation) => observation.runIndex === toolCalls.length, + )?.evidence, + }; + pendingReasoning = ""; + toolCalls.push(call); + byId.set(event.id, call); + } else { + const call = byId.get(event.id); + if (!call) continue; + call.result = event.response ?? event.text; + if (event.image?.data) + call.images = [ + { bytes: Buffer.from(event.image.data, "base64"), mediaType: event.image.mimeType }, + ]; + call.ok = !event.error; + if (event.error) call.error = event.text; + } + } + + return buildTrajectory({ + taskSpec, + toolCalls, + finalAnswer, + status: result.status ?? "complete", + usage: result.usage, + ...(result.finalObservation && { finalObservation: result.finalObservation }), + }); + } +} + +export const geminiCuaAdapter = new GeminiCuaTrajectoryAdapter(); diff --git a/packages/evals/package.json b/packages/evals/package.json index 5743a27c8..b68c26abc 100644 --- a/packages/evals/package.json +++ b/packages/evals/package.json @@ -26,11 +26,13 @@ "@browserbasehq/stagehand": "workspace:*", "@browserbasehq/stagehand-integrations": "workspace:*", "@browserbasehq/stagehand-integrations-claude-agent-sdk": "workspace:*", + "@browserbasehq/stagehand-integrations-claude-cua-sdk": "workspace:*", "@browserbasehq/stagehand-integrations-codex-sdk": "workspace:*", "@browserbasehq/stagehand-integrations-cursor-sdk": "workspace:*", "@browserbasehq/stagehand-integrations-deepagents-sdk": "workspace:*", "@browserbasehq/stagehand-integrations-eve-sdk": "workspace:*", "@browserbasehq/stagehand-integrations-fx-sdk": "workspace:*", + "@browserbasehq/stagehand-integrations-gemini-cua-sdk": "workspace:*", "@browserbasehq/stagehand-integrations-mastra-sdk": "workspace:*", "@browserbasehq/stagehand-integrations-pi-sdk": "workspace:*", "@modelcontextprotocol/sdk": "catalog:", @@ -49,8 +51,7 @@ "stagehand-v3": "npm:@browserbasehq/stagehand@3.7.1", "tsx": "catalog:", "ws": "^8.21.0", - "zod": "catalog:", - "@browserbasehq/stagehand-integrations-claude-cua-sdk": "workspace:*" + "zod": "catalog:" }, "devDependencies": { "@types/node": "catalog:", diff --git a/packages/evals/tests/framework/benchHarness.test.ts b/packages/evals/tests/framework/benchHarness.test.ts index 69128213e..8ea62f8f9 100644 --- a/packages/evals/tests/framework/benchHarness.test.ts +++ b/packages/evals/tests/framework/benchHarness.test.ts @@ -40,6 +40,7 @@ describe("bench harness registry", () => { "fx", "cursor", "claude_cua", + "gemini_cua", ]); }); @@ -47,7 +48,7 @@ describe("bench harness registry", () => { expect(parseBenchHarness(undefined)).toBe("stagehand"); expect(parseBenchHarness("codex")).toBe("codex"); expect(() => parseBenchHarness("nope")).toThrow( - /Unknown harness "nope"\. Supported: stagehand, claude_code, codex, mastra, pi, eve, deepagents, fx, cursor, claude_cua\./, + /Unknown harness "nope"\. Supported: stagehand, claude_code, codex, mastra, pi, eve, deepagents, fx, cursor, claude_cua, gemini_cua\./, ); }); diff --git a/packages/evals/tests/framework/cuaToolAdapter.test.ts b/packages/evals/tests/framework/cuaToolAdapter.test.ts index 4f0dc3c9a..711e7d695 100644 --- a/packages/evals/tests/framework/cuaToolAdapter.test.ts +++ b/packages/evals/tests/framework/cuaToolAdapter.test.ts @@ -1,3 +1,4 @@ +import { GeminiCuaExecutor } from "@browserbasehq/stagehand-integrations-gemini-cua-sdk"; import { StagehandCuaExecutor } from "@browserbasehq/stagehand-integrations-claude-cua-sdk"; import { mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; @@ -76,7 +77,7 @@ import { StagehandFacadeTools } from ${JSON.stringify(compiledFacade)}; let url = 'https://fixture.test', clicks = 0; const page = { pageId:'fixture', url:async()=>url, title:async()=>'Fixture', - goto:async(value)=>{url=value;}, screenshot:async()=>Buffer.from('png'), + goto:async(value)=>{url=value;}, click:async(x,y)=>{url='https://fixture.test/click/'+x+'/'+y;}, setViewportSize:async()=>{}, screenshot:async()=>Buffer.from('png'), evaluate:async(expression)=>{if(expression==='({ width: innerWidth, height: innerHeight })')return {width:1288,height:711};throw new Error('Unexpected fixture evaluate: '+String(expression));}, snapshot:async()=>({formattedTree:'[0-1] button "Submit"',xpathMap:{'0-1':'/button'}}), locator:()=>({click:async()=>{clicks++;}}) @@ -140,6 +141,13 @@ process.stdin.on('end',()=>process.exit(0)); ) ).isError, ).not.toBe(true); + const gemini = new GeminiCuaExecutor(tools, { log() {}, warn() {}, error() {} }); + expect( + (await gemini.execute("navigate", { url: "https://fixture.test/gemini" })).isError, + ).not.toBe(true); + expect(await tools.run("return page.url();")).toBe("https://fixture.test/gemini"); + expect((await gemini.execute("click_at", { x: 500, y: 500 })).isError).not.toBe(true); + expect(await tools.run("return page.url();")).toBe("https://fixture.test/click/644/355"); expect(await tools.screenshot()).toMatchObject({ data: Buffer.from("png").toString("base64"), mimeType: "image/png", diff --git a/packages/evals/tests/framework/geminiCuaAdapter.test.ts b/packages/evals/tests/framework/geminiCuaAdapter.test.ts new file mode 100644 index 000000000..67b0311a3 --- /dev/null +++ b/packages/evals/tests/framework/geminiCuaAdapter.test.ts @@ -0,0 +1,148 @@ +import { + runGeminiCuaSession, + buildGeminiCuaTranscript, +} from "@browserbasehq/stagehand-integrations-gemini-cua-sdk"; +import { vi } from "vitest"; +import { describe, expect, it } from "vitest"; +import type { GeminiCuaSessionEvent } from "@browserbasehq/stagehand-integrations-gemini-cua-sdk"; +import { geminiCuaAdapter } from "../../framework/harnesses/geminiCuaAdapter.js"; + +describe("geminiCuaAdapter", () => { + it("pairs tool events with results and preserves reasoning and observations", () => { + const events: GeminiCuaSessionEvent[] = [ + { + type: "assistant", + turn: 1, + text: "I will click.", + parts: [], + usage: { input: 1, output: 1, reasoning: 0, cached_input: 0, total: 0 }, + }, + { type: "tool_use", turn: 1, id: "call-1", name: "click_at", input: { x: 0, y: 0 } }, + { + type: "tool_result", + turn: 1, + id: "call-1", + name: "click_at", + text: "Clicked.", + error: false, + }, + { + type: "assistant", + turn: 2, + text: 'EVAL_RESULT: {"success":true}', + parts: [], + usage: { input: 1, output: 1, reasoning: 0, cached_input: 0, total: 0 }, + }, + ]; + const trajectory = geminiCuaAdapter.fromHarnessResult( + { events, stepObservations: [{ runIndex: 0, evidence: { url: "https://example.com" } }] }, + { id: "task", instruction: "click", initUrl: "https://example.com" }, + ); + expect(trajectory.steps[0]).toMatchObject({ + actionName: "click_at", + reasoning: "I will click.", + toolOutput: { ok: true, result: "Clicked." }, + probeEvidence: { url: "https://example.com" }, + }); + expect(trajectory.finalAnswer).toContain("EVAL_RESULT"); + }); + it("keeps evidence on its tool ID after an earlier failed call", () => { + const events: GeminiCuaSessionEvent[] = [ + { type: "tool_use", turn: 1, id: "failed", name: "click_at", input: {} }, + { + type: "tool_result", + turn: 1, + id: "failed", + name: "click_at", + text: "missing target", + error: true, + }, + { type: "tool_use", turn: 1, id: "ok", name: "navigate", input: {} }, + { type: "tool_result", turn: 1, id: "ok", name: "navigate", text: "done", error: false }, + ]; + const trajectory = geminiCuaAdapter.fromHarnessResult( + { + events, + stepObservationsByToolUse: new Map([["ok", { url: "https://fixture.test/after" }]]), + }, + { id: "identity", instruction: "navigate" }, + ); + expect(trajectory.steps[0]?.probeEvidence).toEqual({}); + expect(trajectory.steps[1]?.probeEvidence).toEqual({ url: "https://fixture.test/after" }); + }); +}); + +it("preserves the exact model-visible response image and URL under the matching tool ID", async () => { + let turn = 0; + const screenshots = [Buffer.from("first image"), Buffer.from("second image")]; + let screenshotIndex = 0; + const screenshot = vi.fn(async () => ({ + data: screenshots[screenshotIndex++]!.toString("base64"), + mimeType: "image/png" as const, + })); + const requests: Record[] = []; + const result = await runGeminiCuaSession({ + prompt: "p", + model: "gemini-3.8-flash", + logger: { log() {}, warn() {}, error() {} }, + maxTurns: 2, + client: { + generateContent: async (request) => { + requests.push(request); + return ++turn === 1 + ? { + candidates: [ + { + content: { + parts: [ + { functionCall: { id: "first", name: "click_at", args: {} } }, + { functionCall: { id: "second", name: "navigate", args: {} } }, + ], + }, + }, + ], + } + : { candidates: [{ content: { parts: [{ text: "done" }] } }] }; + }, + }, + tools: { execute: async (name) => ({ text: name, isError: name === "click_at" }) }, + facade: { run: async () => "https://fixture.test/visible", screenshot }, + }); + expect(screenshot).toHaveBeenCalledTimes(2); + const recorded = result.events.filter((event) => event.type === "tool_result"); + expect(recorded.map((event) => event.id)).toEqual(["first", "second"]); + const responses = ( + requests[1]!.contents as Array<{ + role: string; + parts: Array<{ + functionResponse?: { + id: string; + response: unknown; + parts: Array<{ inlineData: { data: string; mimeType: string } }>; + }; + }>; + }> + ).find((entry) => entry.role === "user" && entry.parts[0]?.functionResponse)?.parts; + for (let index = 0; index < 2; index++) { + expect(recorded[index]?.response).toEqual(responses?.[index]?.functionResponse?.response); + expect(recorded[index]?.image).toEqual( + responses?.[index]?.functionResponse?.parts[0]?.inlineData, + ); + } + const trajectory = geminiCuaAdapter.fromHarnessResult( + { events: result.events }, + { id: "images", instruction: "act" }, + ); + for (let index = 0; index < 2; index++) + expect(trajectory.steps[index]?.agentEvidence.modalities).toContainEqual({ + type: "image", + bytes: screenshots[index], + mediaType: "image/png", + }); + expect(trajectory.steps[0]?.toolOutput).toMatchObject({ + ok: false, + result: { error: "click_at", url: "https://fixture.test/visible" }, + }); + expect(buildGeminiCuaTranscript(result.events)).toContain("https://fixture.test/visible"); + expect(buildGeminiCuaTranscript(result.events)).toContain("[image: image/png]"); +}); diff --git a/packages/evals/tests/framework/geminiCuaMount.test.ts b/packages/evals/tests/framework/geminiCuaMount.test.ts new file mode 100644 index 000000000..e593cf43e --- /dev/null +++ b/packages/evals/tests/framework/geminiCuaMount.test.ts @@ -0,0 +1,91 @@ +import { geminiCuaHarness } from "../../framework/benchHarness.js"; +import { getCoreTool, listCoreRunnableTools } from "../../core/tools/registry.js"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + prepareGeminiCuaToolAdapter, + GEMINI_CUA_TOOL_INSTRUCTIONS, +} from "../../framework/geminiCuaToolAdapter.js"; +import { startAgentToolRuntime } from "../../framework/agentToolRuntime.js"; +import { EvalLogger } from "../../logger.js"; + +vi.mock("../../framework/agentToolRuntime.js", () => ({ startAgentToolRuntime: vi.fn() })); +afterEach(() => vi.unstubAllEnvs()); + +const input = { + environment: "LOCAL" as const, + plan: { + dataset: "webvoyager" as const, + taskId: "native-mount", + startUrl: "https://fixture.test", + instruction: "Finish the task.", + }, + logger: new EvalLogger(false), +}; + +describe("Gemini native mount ownership", () => { + it("uses the supplied browser and typed bridge, preserves loss metadata, and cleans up once", async () => { + const cleanup = vi.fn(async () => {}); + const loss = (): undefined => undefined; + const callTool = vi.fn(async () => ({ + content: [{ type: "text", text: "{}" }], + })); + const browserSession = { provider: "local" as const, sessionId: "owned-fixture" }; + vi.mocked(startAgentToolRuntime).mockResolvedValue({ + browserSession, + running: { + agentMount: { + via: "mcp", + promptInstructions: + "You control one persistent browser through exactly three tools: run, snapshot, screenshot.", + mcpServers: {}, + }, + callTool, + browserSessionLoss: loss, + } as never, + cleanup, + }); + const adapter = await prepareGeminiCuaToolAdapter(input); + expect(adapter.promptInstructions).toBe(GEMINI_CUA_TOOL_INSTRUCTIONS); + expect(adapter.promptInstructions).not.toContain("exactly three tools"); + expect(adapter.promptInstructions).not.toContain("run, snapshot, screenshot"); + expect(adapter.browserSession).toBe(browserSession); + expect(adapter.browserSessionLoss).toBe(loss); + expect( + ( + await adapter.executor.execute( + "navigate", + { url: "https://fixture.test" }, + { toolUseId: "read" }, + ) + ).isError, + ).not.toBe(true); + expect(callTool).toHaveBeenCalledWith( + "run", + { code: expect.stringContaining('page.goto("https://fixture.test"') }, + { timeoutMs: 90_000 }, + ); + await Promise.all([adapter.cleanup(), adapter.cleanup()]); + expect(cleanup).toHaveBeenCalledOnce(); + }); + + it("closes the same runtime if it cannot expose native facade calls", async () => { + const cleanup = vi.fn(async () => {}); + vi.mocked(startAgentToolRuntime).mockResolvedValue({ + browserSession: { provider: "local" }, + running: {} as never, + cleanup, + }); + await expect(prepareGeminiCuaToolAdapter(input)).rejects.toThrow( + "does not expose runner-side tool calls", + ); + expect(cleanup).toHaveBeenCalledOnce(); + }); +}); + +it("registers native Gemini only on the facade-backed Google computer-use surface", () => { + expect(geminiCuaHarness.supportedToolSurfaces).toEqual(["google_computer_use"]); + expect(geminiCuaHarness.defaultModels).toEqual(["google/gemini-3.8-flash"]); + expect(geminiCuaHarness.execute).toBeTypeOf("function"); + expect(getCoreTool("google_computer_use").id).toBe("google_computer_use"); + expect(listCoreRunnableTools()).not.toContain("google_computer_use"); +}); diff --git a/packages/evals/tests/framework/geminiCuaRunner.test.ts b/packages/evals/tests/framework/geminiCuaRunner.test.ts new file mode 100644 index 000000000..3aa32a426 --- /dev/null +++ b/packages/evals/tests/framework/geminiCuaRunner.test.ts @@ -0,0 +1,83 @@ +import { GEMINI_CUA_TOOL_INSTRUCTIONS } from "../../framework/geminiCuaToolAdapter.js"; +import { describe, expect, it } from "vitest"; +import type { GeminiGenerateClient } from "@browserbasehq/stagehand-integrations-gemini-cua-sdk"; +import type { AvailableModel } from "stagehand-v3"; +import { runGeminiCuaAgent, assertGeminiCuaModel } from "../../framework/geminiCuaRunner.js"; +import { EVAL_SYSTEM_PROMPT } from "../../framework/evalSystemPrompt.js"; +import { EvalLogger } from "../../logger.js"; + +describe("Gemini CUA eval system prompt", () => { + it("sends the common policy once through the native system channel", async () => { + let request: Record | undefined; + const client: GeminiGenerateClient = { + generateContent: async (params) => { + request = params; + return { + candidates: [ + { + content: { parts: [{ text: 'EVAL_RESULT: {"success":true}' }] }, + finishReason: "STOP", + }, + ], + usageMetadata: { promptTokenCount: 10, candidatesTokenCount: 5 }, + }; + }, + }; + const instruction = "Find the item and stop before the final purchase."; + const result = await runGeminiCuaAgent({ + plan: { + dataset: "webvoyager", + taskId: "cua-system-prompt", + startUrl: "https://example.com", + instruction, + }, + model: "google/gemini-3.8-flash" as AvailableModel, + logger: new EvalLogger(false), + client, + toolAdapter: { + toolSurface: "google_computer_use", + startupProfile: "tool_launch_local", + browserSession: { provider: "local" }, + promptInstructions: GEMINI_CUA_TOOL_INSTRUCTIONS, + executor: { + execute: async () => { + throw new Error("This recording client does not request tools."); + }, + }, + facade: { + run: async () => "", + screenshot: async () => ({ data: "png", mimeType: "image/png" }), + }, + observedToolMatcher: () => true, + cleanup: async () => {}, + }, + }); + + expect(result._success).toBe(true); + expect( + String((request?.config as { systemInstruction?: string })?.systemInstruction).split( + EVAL_SYSTEM_PROMPT, + ), + ).toHaveLength(2); + expect((request?.config as { systemInstruction?: string })?.systemInstruction).toContain( + "computer use only", + ); + expect((request?.config as { systemInstruction?: string })?.systemInstruction).toContain( + "requested EVAL_RESULT line", + ); + const messages = JSON.stringify(request?.contents); + expect(messages).not.toContain(EVAL_SYSTEM_PROMPT); + expect(messages).toContain(instruction); + expect(messages).toContain("Browser tool surface: Gemini Computer Use."); + expect(messages).not.toContain("Stagehand Playwright facade"); + expect(messages).not.toContain("exactly three tools"); + expect(messages).not.toContain("run, snapshot, screenshot"); + }); +}); + +it("rejects models from a different provider", () => { + expect(() => assertGeminiCuaModel("google/gemini-3.8-flash")).not.toThrow(); + expect(() => assertGeminiCuaModel("anthropic/gemini-3.8-flash")).toThrow( + "Google Gemini models only", + ); +}); diff --git a/packages/integrations/gemini-cua-sdk/ARCHITECTURE.md b/packages/integrations/gemini-cua-sdk/ARCHITECTURE.md new file mode 100644 index 000000000..2c48bfd09 --- /dev/null +++ b/packages/integrations/gemini-cua-sdk/ARCHITECTURE.md @@ -0,0 +1,15 @@ +# gemini-cua-sdk + +This package runs Gemini Computer Use against a Stagehand facade browser. The +session owns the `generateContent` loop and transcript; the executor translates +the provider's coordinate and keyboard actions into facade `run` calls. Each +response cycle returns one fresh screenshot and the current URL as a +`functionResponse`, and provider usage metadata is retained per step. + +The evals `gemini_cua` harness mounts only `google_computer_use`; it uses the same runner-owned facade bridge and browser identity as other mounted harnesses. The native SDK calls the canonical facade capability types through the shared CUA decoder, not a separate browser implementation. + +The common evaluation policy is sent once via `config.systemInstruction`, the installed Google SDK request field. `config.abortSignal` cancels the active client request; aborts also stop retry waits and are checked before browser initialization and between actions. A terminal browser loss ends the session without retrying screenshots. Transient observation failures retain the existing one-retry behavior. Eval cleanup is bounded and idempotent. + +Budgets count API turns, which can contain multiple tool calls. The exact per-action response and image sent to the model are retained on the tool-result event and attached by tool-use ID, so failed calls do not shift observations. The eval adapter uses these returned images rather than capturing another screenshot for each action. Historical index-based observations remain readable by the trajectory adapter. Input/cache/output/reasoning counters retain the provider's separate fields for shared usage normalization. + +Coordinate actions use a 1288×711 viewport and Gemini's normalized coordinates. The executor supports the provider actions listed in its switch; unknown actions return explicit errors. It does not add facade reconnection or alternate browser ownership. diff --git a/packages/integrations/gemini-cua-sdk/package.json b/packages/integrations/gemini-cua-sdk/package.json new file mode 100644 index 000000000..da1146fce --- /dev/null +++ b/packages/integrations/gemini-cua-sdk/package.json @@ -0,0 +1,36 @@ +{ + "name": "@browserbasehq/stagehand-integrations-gemini-cua-sdk", + "version": "4.0.1", + "private": true, + "description": "Google Gemini Computer Use harness adapter for Stagehand integrations", + "files": [ + "dist" + ], + "type": "module", + "exports": { + ".": { + "types": "./dist/index.d.mts", + "import": "./dist/index.mjs" + } + }, + "scripts": { + "build": "tsdown", + "test": "pnpm run build && vitest run --root ../../.. packages/integrations/gemini-cua-sdk/tests", + "test:unit": "vitest run --root ../../.. packages/integrations/gemini-cua-sdk/tests", + "typecheck": "tsc --noEmit -p tsconfig.json" + }, + "dependencies": { + "@browserbasehq/stagehand": "workspace:*", + "@browserbasehq/stagehand-integrations": "workspace:*", + "@google/genai": "catalog:" + }, + "devDependencies": { + "@types/node": "catalog:", + "tsdown": "catalog:", + "typescript": "catalog:", + "vitest": "catalog:" + }, + "engines": { + "node": ">=22.18.0" + } +} diff --git a/packages/integrations/gemini-cua-sdk/src/executor.ts b/packages/integrations/gemini-cua-sdk/src/executor.ts new file mode 100644 index 000000000..23ab30e02 --- /dev/null +++ b/packages/integrations/gemini-cua-sdk/src/executor.ts @@ -0,0 +1,285 @@ +import { + isBrowserSessionLostError, + type StagehandFacadeTools, +} from "@browserbasehq/stagehand-integrations/facade"; +import type { HarnessLogger } from "@browserbasehq/stagehand-integrations/harness"; + +export type CuaFacadeTools = Pick; + +export type GeminiToolResult = { text: string; isError?: boolean }; +export type GeminiToolExecutor = { + execute( + name: string, + input: Record, + context?: { signal?: AbortSignal; toolUseId?: string }, + ): Promise; +}; + +const WIDTH = 1288; +const HEIGHT = 711; +const KEY_NAMES: Record = { + enter: "Enter", + return: "Enter", + esc: "Escape", + escape: "Escape", + tab: "Tab", + backspace: "Backspace", + delete: "Delete", + space: " ", + up: "ArrowUp", + down: "ArrowDown", + left: "ArrowLeft", + right: "ArrowRight", + ctrl: "Control", + control: "Control", + alt: "Alt", + shift: "Shift", + cmd: "Meta", + command: "Meta", + meta: "Meta", + pageup: "PageUp", + pagedown: "PageDown", +}; + +export function denormalizeCoordinate(value: unknown, size: number): number { + const number = typeof value === "number" && Number.isFinite(value) ? value : 0; + return Math.floor((Math.max(0, Math.min(999, number)) / 1000) * size); +} + +export class GeminiCuaExecutor implements GeminiToolExecutor { + constructor( + private readonly tools: CuaFacadeTools, + private readonly logger: HarnessLogger, + private readonly onMutation?: (toolUseId: string) => Promise, + ) {} + + async execute( + name: string, + input: Record, + context: { signal?: AbortSignal; toolUseId?: string } = {}, + ): Promise { + const result = await this.executeAction(name, input, context); + if (!result.isError && context.toolUseId && this.onMutation) { + await this.onMutation(context.toolUseId).catch((error: unknown) => { + if (isBrowserSessionLostError(error instanceof Error ? error.message : String(error))) + throw error; + }); + } + return result; + } + + private async executeAction( + name: string, + input: Record, + context: { signal?: AbortSignal } = {}, + ): Promise { + try { + if (context.signal?.aborted) throw new Error("operation aborted"); + switch (name) { + case "click_at": + case "click": + case "left_click": + return await this.click(input, "left", 1); + case "double_click": + return await this.click(input, "left", 2); + case "triple_click": + return await this.click(input, "left", 3); + case "right_click": + return await this.click(input, "right", 1); + case "middle_click": + return await this.click(input, "middle", 1); + case "move": + case "hover_at": + case "hover": + return await this.hover(input); + case "type_text_at": + case "type": + case "type_text": + return await this.type(input); + case "key_combination": + case "key": + case "keys": + case "key_press": + case "press_key": + case "press_keys": + case "hotkey": + return await this.pressKeys(input); + case "scroll_document": + case "scroll_at": + case "scroll": + return await this.scroll(name, input); + case "navigate": + return await this.navigate(input); + case "go_back": + await this.runWithTabs("await page.goBack();"); + return { text: "Went back." }; + case "go_forward": + await this.runWithTabs("await page.goForward();"); + return { text: "Went forward." }; + case "wait_5_seconds": + case "wait": + await this.runWithTabs("await page.waitForTimeout(5000);"); + return { text: "Waited 5 seconds." }; + case "take_screenshot": + case "screenshot": + return { text: "Screenshot captured." }; + case "open_web_browser": + return { text: "Browser is already open." }; + case "search": + await this.runWithTabs('await page.goto("https://www.google.com");'); + return { text: "Opened Google search." }; + case "drag_and_drop": + case "drag": + return await this.drag(input); + default: + return { text: `Error: unknown computer-use action "${name}".`, isError: true }; + } + } catch (error) { + if ( + context.signal?.aborted || + isBrowserSessionLostError(error instanceof Error ? error.message : String(error)) + ) + throw error; + const message = error instanceof Error ? error.message : String(error); + this.logger.warn({ category: "gemini_cua", message: `${name} failed: ${message}`, level: 1 }); + return { text: `Error: ${message}`, isError: true }; + } + } + + private async click( + input: Record, + button: "left" | "middle" | "right", + clickCount: number, + ): Promise { + const [x, y] = this.requiredCoordinate(input); + await this.runWithTabs( + `await batchStagehand.page.click(${x}, ${y}, ${JSON.stringify({ button, clickCount })});`, + ); + return { text: `Clicked at (${x}, ${y}).` }; + } + + private async hover(input: Record): Promise { + const [x, y] = this.requiredCoordinate(input); + await this.runWithTabs(`await batchStagehand.page.hover(${x}, ${y});`); + return { text: `Moved to (${x}, ${y}).` }; + } + + private async type(input: Record): Promise { + const lines: string[] = []; + const hasCoordinates = typeof input.x === "number" && typeof input.y === "number"; + if (hasCoordinates) { + const [x, y] = this.requiredCoordinate(input); + lines.push(`await batchStagehand.page.click(${x}, ${y});`); + } + if ( + input.clear_before_typing === true || + (hasCoordinates && input.clear_before_typing !== false) + ) { + lines.push( + 'await batchStagehand.page.keyPress("Control+a");', + 'await batchStagehand.page.keyPress("Backspace");', + ); + } + lines.push(`await batchStagehand.page.type(${JSON.stringify(String(input.text ?? ""))});`); + if (input.press_enter === true) lines.push('await batchStagehand.page.keyPress("Enter");'); + await this.runWithTabs(lines.join("\n")); + return { text: `Typed ${JSON.stringify(String(input.text ?? ""))}.` }; + } + + private async pressKeys(input: Record): Promise { + const raw = input.keys ?? input.key ?? input.text; + const values = Array.isArray(raw) + ? raw + : String(raw ?? "") + .split(/\s+/u) + .filter(Boolean); + if (values.length === 0) return { text: "Error: no keys supplied.", isError: true }; + const pressed = values.map((value) => this.chord(value)); + await this.runWithTabs( + pressed + .map((key) => `await batchStagehand.page.keyPress(${JSON.stringify(key)});`) + .join("\n"), + ); + return { text: `Pressed ${pressed.join(", ")}.` }; + } + + private async scroll(name: string, input: Record): Promise { + const direction = String(input.direction ?? "down").toLowerCase(); + if (name === "scroll_document" || (input.x === undefined && input.y === undefined)) { + await this.runWithTabs( + `await batchStagehand.page.keyPress(${JSON.stringify(direction === "up" ? "PageUp" : "PageDown")});`, + ); + return { text: "Scrolled." }; + } + const [x, y] = this.requiredCoordinate(input); + const amount = + typeof input.magnitude === "number" + ? input.magnitude + : typeof input.magnitude_in_pixels === "number" + ? input.magnitude_in_pixels + : 800; + const deltaX = direction === "left" ? -amount : direction === "right" ? amount : 0; + const deltaY = direction === "up" ? -amount : direction === "down" ? amount : 0; + await this.runWithTabs(`await batchStagehand.page.scroll(${x}, ${y}, ${deltaX}, ${deltaY});`); + return { text: "Scrolled." }; + } + + private async navigate(input: Record): Promise { + const url = String(input.url ?? "").trim(); + if (!/^https?:\/\//u.test(url)) + return { text: "Error: navigate requires an http(s) URL.", isError: true }; + await this.runWithTabs( + `await page.goto(${JSON.stringify(url)}, { waitUntil: "domcontentloaded" });`, + ); + return { text: `Navigated to ${url}.` }; + } + + private async drag(input: Record): Promise { + const values = [ + input.start_x ?? input.x, + input.start_y ?? input.y, + input.end_x ?? input.destination_x, + input.end_y ?? input.destination_y, + ]; + if (!values.every((value) => typeof value === "number" && Number.isFinite(value))) + throw new Error("drag requires four finite coordinates"); + const start = [ + denormalizeCoordinate(values[0], WIDTH), + denormalizeCoordinate(values[1], HEIGHT), + ]; + const end = [denormalizeCoordinate(values[2], WIDTH), denormalizeCoordinate(values[3], HEIGHT)]; + await this.runWithTabs( + `const __p = batchStagehand.page; +await __p.sendCDP("Input.dispatchMouseEvent", { type: "mouseMoved", x: ${start[0]}, y: ${start[1]}, button: "none" }); +await __p.sendCDP("Input.dispatchMouseEvent", { type: "mousePressed", x: ${start[0]}, y: ${start[1]}, button: "left", clickCount: 1 }); +await __p.sendCDP("Input.dispatchMouseEvent", { type: "mouseMoved", x: ${(start[0] + end[0]) / 2}, y: ${(start[1] + end[1]) / 2}, button: "left" }); +await __p.sendCDP("Input.dispatchMouseEvent", { type: "mouseMoved", x: ${end[0]}, y: ${end[1]}, button: "left" }); +await __p.sendCDP("Input.dispatchMouseEvent", { type: "mouseReleased", x: ${end[0]}, y: ${end[1]}, button: "left", clickCount: 1 });`, + ); + return { text: `Dragged from (${start[0]}, ${start[1]}) to (${end[0]}, ${end[1]}).` }; + } + + private requiredCoordinate(input: Record): [number, number] { + if ( + typeof input.x !== "number" || + !Number.isFinite(input.x) || + typeof input.y !== "number" || + !Number.isFinite(input.y) + ) + throw new Error("missing or invalid coordinates (x, y)"); + return [denormalizeCoordinate(input.x, WIDTH), denormalizeCoordinate(input.y, HEIGHT)]; + } + + private chord(value: unknown): string { + return String(value) + .split("+") + .map((part) => KEY_NAMES[part.toLowerCase()] ?? part) + .join("+"); + } + + private async runWithTabs(code: string): Promise { + return await this.tools.run(code); + } +} + +export const GEMINI_CUA_VIEWPORT = { width: WIDTH, height: HEIGHT }; diff --git a/packages/integrations/gemini-cua-sdk/src/index.ts b/packages/integrations/gemini-cua-sdk/src/index.ts new file mode 100644 index 000000000..41989ac92 --- /dev/null +++ b/packages/integrations/gemini-cua-sdk/src/index.ts @@ -0,0 +1,3 @@ +export * from "./executor.js"; +export * from "./session.js"; +export * from "./transcript.js"; diff --git a/packages/integrations/gemini-cua-sdk/src/session.ts b/packages/integrations/gemini-cua-sdk/src/session.ts new file mode 100644 index 000000000..eb94d91be --- /dev/null +++ b/packages/integrations/gemini-cua-sdk/src/session.ts @@ -0,0 +1,337 @@ +import { isBrowserSessionLostError } from "@browserbasehq/stagehand-integrations/facade"; +import { GoogleGenAI, Environment, type GenerateContentConfig } from "@google/genai"; +import { + sanitizeErrorMessage, + type HarnessLogger, +} from "@browserbasehq/stagehand-integrations/harness"; +import type { CuaFacadeTools, GeminiToolExecutor } from "./executor.js"; + +export type GeminiCuaUsage = { + input: number; + output: number; + reasoning: number; + cached_input: number; + total: number; +}; +export type GeminiCuaSessionEvent = + | { type: "assistant"; turn: number; text: string; parts: unknown[]; usage: GeminiCuaUsage } + | { type: "tool_use"; turn: number; id: string; name: string; input: Record } + | { + type: "tool_result"; + turn: number; + id: string; + name: string; + text: string; + error: boolean; + response?: Record; + image?: { data: string; mimeType: string }; + }; +export type GeminiGenerateClient = { + generateContent(request: Record): Promise; +}; +export type GeminiCuaSessionInput = { + prompt: string; + model: string; + logger: HarnessLogger; + maxTurns: number; + tools: GeminiToolExecutor; + facade: Pick; + systemPrompt?: string; + signal?: AbortSignal; + client?: GeminiGenerateClient; +}; +export type GeminiCuaSessionResult = { + events: GeminiCuaSessionEvent[]; + finalMessage: string; + status: "completed" | "max_turns" | "sdk_error"; + stopReason?: string; + tokenUsage: GeminiCuaUsage; + /** Whether any response supplied a finite nonnegative token counter (including real zero). */ + usageReported: boolean; + turns: number; + toolCalls: number; + iterationError?: unknown; +}; + +export const GEMINI_CUA_DEFAULT_MODEL = "gemini-3.8-flash"; +export const GEMINI_CUA_GENERATION_CONFIG = { + temperature: 1, + topP: 0.95, + topK: 40, + maxOutputTokens: 8192, +} as const; + +export function normalizeGeminiCuaModel(model: string): string { + const bare = model.includes("/") ? model.slice(model.indexOf("/") + 1) : model; + const provider = model.includes("/") ? model.slice(0, model.indexOf("/")) : undefined; + if ((provider !== undefined && provider !== "google") || !bare.startsWith("gemini-")) + throw new Error(`Gemini Computer Use requires a gemini-* model (received "${model}")`); + return bare; +} + +export function createGeminiClient(options?: { apiKey?: string }): GeminiGenerateClient { + const client = new GoogleGenAI({ ...(options?.apiKey && { apiKey: options.apiKey }) }); + return { + generateContent: (request) => + client.models.generateContent(request as never) as unknown as Promise, + }; +} + +export async function runGeminiCuaSession( + input: GeminiCuaSessionInput, +): Promise { + const model = normalizeGeminiCuaModel(input.model); + const contents: Array> = [ + { role: "user", parts: [{ text: input.prompt }] }, + ]; + const events: GeminiCuaSessionEvent[] = []; + let usageReported = false; + const usage: GeminiCuaUsage = { input: 0, output: 0, reasoning: 0, cached_input: 0, total: 0 }; + let finalMessage = ""; + let turns = 0; + let toolCalls = 0; + let status: GeminiCuaSessionResult["status"] = "max_turns"; + let stopReason: string | undefined; + let iterationError: unknown; + + try { + throwIfAborted(input.signal); + const client = input.client ?? createGeminiClient(); + await input.facade.run("await page.setViewportSize({ width: 1288, height: 711 });"); + while (turns < input.maxTurns) { + throwIfAborted(input.signal); + turns += 1; + const response = await generateWithRetry( + client, + { + model, + contents, + config: { + ...(input.systemPrompt && { systemInstruction: input.systemPrompt }), + ...(input.signal && { abortSignal: input.signal }), + ...GEMINI_CUA_GENERATION_CONFIG, + tools: [{ computerUse: { environment: Environment.ENVIRONMENT_BROWSER } }], + } satisfies GenerateContentConfig, + }, + input.signal, + ); + const candidate = ( + response as { + candidates?: Array<{ content?: { parts?: unknown[] }; finishReason?: string }>; + } + ).candidates?.[0]; + const parts = candidate?.content?.parts ?? []; + const rawUsage = + (response as { usageMetadata?: Record }).usageMetadata ?? {}; + usageReported ||= [ + "promptTokenCount", + "candidatesTokenCount", + "thoughtsTokenCount", + "cachedContentTokenCount", + "totalTokenCount", + ].some( + (key) => + typeof rawUsage[key] === "number" && + Number.isFinite(rawUsage[key]) && + Number(rawUsage[key]) >= 0, + ); + const turnUsage = readUsage(rawUsage); + usage.input += turnUsage.input; + usage.output += turnUsage.output; + usage.reasoning += turnUsage.reasoning; + usage.cached_input += turnUsage.cached_input; + usage.total += turnUsage.total; + const text = parts + .filter(isRecord) + .map((part) => (typeof part.text === "string" ? part.text : "")) + .filter(Boolean) + .join("\n") + .trim(); + events.push({ type: "assistant", turn: turns, text, parts, usage: turnUsage }); + contents.push({ role: "model", parts }); + const calls = parts + .filter(isRecord) + .map((part) => part.functionCall) + .filter(isRecord); + if (calls.length === 0) { + finalMessage = text; + status = "completed"; + stopReason = candidate?.finishReason; + break; + } + const results: Record[] = []; + for (const call of calls) { + throwIfAborted(input.signal); + const name = typeof call.name === "string" ? call.name : ""; + const args = isRecord(call.args) ? call.args : {}; + const id = typeof call.id === "string" ? call.id : `call_${toolCalls}`; + toolCalls += 1; + events.push({ type: "tool_use", turn: turns, id, name, input: args }); + const result = await input.tools.execute(name, args, { + signal: input.signal, + toolUseId: id, + }); + const resultEvent: Extract = { + type: "tool_result", + turn: turns, + id, + name, + text: result.text, + error: result.isError === true, + }; + events.push(resultEvent); + const observed = await observePage(input.facade, input.signal); + const response: Record = result.isError + ? { error: result.text } + : { output: result.text }; + response.url = observed.url; + if (observed.error) response.screenshot_error = observed.error; + if (args.safety_decision !== undefined) response.safety_acknowledgement = "true"; + resultEvent.response = response; + if (observed.shot) resultEvent.image = observed.shot; + results.push({ + functionResponse: { + id, + name, + response, + ...(observed.shot && { + parts: [ + { inlineData: { mimeType: observed.shot.mimeType, data: observed.shot.data } }, + ], + }), + }, + }); + } + contents.push({ role: "user", parts: results }); + } + } catch (error) { + status = "sdk_error"; + iterationError = error; + stopReason = sanitizeErrorMessage(error instanceof Error ? error.message : String(error)); + input.logger.warn({ + category: "gemini_cua", + message: `session ended with error: ${stopReason}`, + level: 1, + }); + } + return { + events, + finalMessage, + status, + stopReason, + tokenUsage: usage, + usageReported, + turns, + toolCalls, + ...(iterationError !== undefined && { iterationError }), + }; +} + +async function generateWithRetry( + client: GeminiGenerateClient, + request: Record, + signal?: AbortSignal, +): Promise { + let last: unknown; + for (let attempt = 0; attempt < 5; attempt += 1) { + try { + throwIfAborted(signal); + return await client.generateContent(request); + } catch (error) { + if (signal?.aborted) throw error; + last = error; + if (!isTransientError(error) || attempt === 4) break; + const delay = 100 * 2 ** attempt + Math.floor(Math.random() * 51); + await pause(delay, signal); + } + } + throw last; +} + +function isTransientError(error: unknown): boolean { + const value = error as { + status?: unknown; + statusCode?: unknown; + code?: unknown; + message?: unknown; + }; + const status = Number(value.status ?? value.statusCode ?? value.code); + return ( + status === 429 || + (status >= 500 && status <= 599) || + /network|timeout|fetch failed|econn/iu.test(String(value.message ?? error)) + ); +} + +function readUsage(meta: Record): GeminiCuaUsage { + const number = (key: string) => + typeof meta[key] === "number" && Number.isFinite(meta[key]) ? Number(meta[key]) : 0; + const reasoning = number("thoughtsTokenCount"); + return { + input: number("promptTokenCount"), + output: number("candidatesTokenCount"), + reasoning, + cached_input: number("cachedContentTokenCount"), + total: + typeof meta.totalTokenCount === "number" && + Number.isFinite(meta.totalTokenCount) && + meta.totalTokenCount >= 0 + ? meta.totalTokenCount + : number("promptTokenCount") + number("candidatesTokenCount") + reasoning, + }; +} + +function throwIfAborted(signal?: AbortSignal): void { + if (signal?.aborted) throw new Error("session aborted"); +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null; +} + +/** + * Post-action observation for the model. A page mid-navigation can make the + * facade's screenshot / url calls fail (e.g. waitForMainLoadState timeouts); + * that is a transient page state, not a session failure, so retry once and + * otherwise tell the model the screenshot is unavailable this turn. + */ +async function observePage( + facade: Pick, + signal?: AbortSignal, +): Promise<{ shot?: { data: string; mimeType: string }; url: string; error?: string }> { + let lastError = ""; + for (let attempt = 0; attempt < 2; attempt += 1) { + try { + throwIfAborted(signal); + const shot = await facade.screenshot({ type: "png" }); + const urlValue = await facade.run("return page.url();"); + return { shot, url: typeof urlValue === "string" ? urlValue : "" }; + } catch (error) { + if ( + signal?.aborted || + isBrowserSessionLostError(error instanceof Error ? error.message : String(error)) + ) + throw error; + lastError = sanitizeErrorMessage(error instanceof Error ? error.message : String(error)); + if (attempt === 0) await pause(1500, signal); + } + } + return { url: "", error: `screenshot unavailable this turn: ${lastError}` }; +} + +function pause(ms: number, signal?: AbortSignal): Promise { + throwIfAborted(signal); + return new Promise((resolve, reject) => { + const done = () => { + signal?.removeEventListener("abort", aborted); + resolve(); + }; + const timer = setTimeout(done, ms); + const aborted = () => { + clearTimeout(timer); + signal?.removeEventListener("abort", aborted); + reject(new Error("session aborted")); + }; + signal?.addEventListener("abort", aborted, { once: true }); + }); +} diff --git a/packages/integrations/gemini-cua-sdk/src/transcript.ts b/packages/integrations/gemini-cua-sdk/src/transcript.ts new file mode 100644 index 000000000..324d51cee --- /dev/null +++ b/packages/integrations/gemini-cua-sdk/src/transcript.ts @@ -0,0 +1,25 @@ +import type { GeminiCuaSessionEvent } from "./session.js"; + +const MAX_RESULT_CHARS = 2_000; + +export function buildGeminiCuaTranscript(events: GeminiCuaSessionEvent[]): string { + const lines: string[] = []; + for (const event of events) { + if (event.type === "assistant") { + if (event.text) lines.push(event.text); + } else if (event.type === "tool_use") { + lines.push(`${JSON.stringify(event.input)}`); + } else { + const status = event.error ? ' status="error"' : ""; + lines.push( + `${clip(event.response ? JSON.stringify(event.response) : event.text, MAX_RESULT_CHARS)}`, + ); + if (event.image) lines.push(`[image: ${event.image.mimeType}]`); + } + } + return lines.join("\n"); +} + +function clip(value: string, max: number): string { + return value.length <= max ? value : `${value.slice(0, max - 1)}…`; +} diff --git a/packages/integrations/gemini-cua-sdk/tests/executor.test.ts b/packages/integrations/gemini-cua-sdk/tests/executor.test.ts new file mode 100644 index 000000000..87b445ba9 --- /dev/null +++ b/packages/integrations/gemini-cua-sdk/tests/executor.test.ts @@ -0,0 +1,130 @@ +import { describe, expect, it, vi } from "vitest"; +import { GeminiCuaExecutor } from "../src/executor.js"; +const logger = { log: () => {}, warn: () => {}, error: () => {} }; +describe("GeminiCuaExecutor", () => { + it("denormalizes coordinates and expands type_text_at", async () => { + const calls: string[] = []; + const executor = new GeminiCuaExecutor( + { + run: async (code) => { + calls.push(code); + }, + screenshot: async () => ({ data: "", mimeType: "image/png" }), + }, + logger, + ); + await executor.execute("type_text_at", { x: 500, y: 500, text: "hello", press_enter: true }); + expect(calls[0]).toContain("page.click(644, 355"); + expect(calls[0]).toContain('page.keyPress("Control+a")'); + expect(calls[0]).toContain('page.keyPress("Backspace")'); + expect(calls[0]).toContain('page.keyPress("Enter")'); + }); + + it.each([ + [0, 0, 0, 0], + [999, 999, 1286, 710], + [-10, 2000, 0, 710], + ])("clamps normalized coordinates (%s, %s)", async (inputX, inputY, outputX, outputY) => { + const calls: string[] = []; + const executor = new GeminiCuaExecutor( + { + run: async (code) => calls.push(code), + screenshot: async () => ({ data: "", mimeType: "image/png" }), + }, + logger, + ); + await executor.execute("click_at", { x: inputX, y: inputY }); + expect(calls[0]).toContain(`page.click(${outputX}, ${outputY}`); + }); + + it("emits the action aliases and exact runtime calls", async () => { + const calls: string[] = []; + const executor = new GeminiCuaExecutor( + { + run: async (code) => calls.push(code), + screenshot: async () => ({ data: "", mimeType: "image/png" }), + }, + logger, + ); + await executor.execute("scroll_at", { x: 500, y: 500, direction: "right" }); + await executor.execute("scroll_document", { direction: "up" }); + await executor.execute("search", {}); + await executor.execute("drag_and_drop", { x: 0, y: 0, destination_x: 999, destination_y: 999 }); + expect(calls[0]).toContain("page.scroll(644, 355, 800, 0)"); + expect(calls[1]).toContain('page.keyPress("PageUp")'); + expect(calls[2]).toContain('page.goto("https://www.google.com")'); + expect(calls[3]).toContain( + 'await __p.sendCDP("Input.dispatchMouseEvent", { type: "mouseMoved", x: 0, y: 0, button: "none" });', + ); + expect(calls[3]).toContain( + 'await __p.sendCDP("Input.dispatchMouseEvent", { type: "mousePressed", x: 0, y: 0, button: "left", clickCount: 1 });', + ); + expect(calls[3]).toContain( + 'await __p.sendCDP("Input.dispatchMouseEvent", { type: "mouseMoved", x: 643, y: 355, button: "left" });', + ); + expect(calls[3]).toContain( + 'await __p.sendCDP("Input.dispatchMouseEvent", { type: "mouseMoved", x: 1286, y: 710, button: "left" });', + ); + expect(calls[3]).toContain( + 'await __p.sendCDP("Input.dispatchMouseEvent", { type: "mouseReleased", x: 1286, y: 710, button: "left", clickCount: 1 });', + ); + }); + + it("supports focused typing without clearing and rejects unknown navigation", async () => { + const calls: string[] = []; + const executor = new GeminiCuaExecutor( + { + run: async (code) => calls.push(code), + screenshot: async () => ({ data: "", mimeType: "image/png" }), + }, + logger, + ); + await executor.execute("type_text_at", { text: "x", clear_before_typing: false }); + const result = await executor.execute("navigate", { url: "javascript:bad" }); + expect(calls[0]).toBe('await batchStagehand.page.type("x");'); + expect(result.isError).toBe(true); + }); + it("returns an error for unknown actions", async () => { + expect( + ( + await new GeminiCuaExecutor( + { run: async () => {}, screenshot: async () => ({ data: "", mimeType: "image/png" }) }, + logger, + ).execute("unknown", {}) + ).isError, + ).toBe(true); + }); +}); + +describe("Gemini executor failure lifecycle", () => { + const logger = { log() {}, warn() {}, error() {} }; + it("returns recoverable asynchronous action errors while propagating terminal loss", async () => { + const tools = { + run: vi.fn(async () => { + throw new Error("target missing"); + }), + screenshot: async () => ({ data: "png", mimeType: "image/png" as const }), + }; + const executor = new GeminiCuaExecutor(tools, logger); + expect(await executor.execute("click_at", { x: 100, y: 100 })).toEqual({ + text: "Error: target missing", + isError: true, + }); + tools.run.mockRejectedValue(new Error("Browser session lost (closed). Stop.")); + await expect(executor.execute("click_at", { x: 100, y: 100 })).rejects.toThrow( + "Browser session lost", + ); + }); + it("propagates terminal loss from the observation hook", async () => { + const tools = { + run: async () => ({}), + screenshot: async () => ({ data: "png", mimeType: "image/png" as const }), + }; + const executor = new GeminiCuaExecutor(tools, logger, async () => { + throw new Error("Browser session lost (closed). Stop."); + }); + await expect( + executor.execute("navigate", { url: "https://fixture.test" }, { toolUseId: "nav" }), + ).rejects.toThrow("Browser session lost"); + }); +}); diff --git a/packages/integrations/gemini-cua-sdk/tests/session.test.ts b/packages/integrations/gemini-cua-sdk/tests/session.test.ts new file mode 100644 index 000000000..a10c8a82f --- /dev/null +++ b/packages/integrations/gemini-cua-sdk/tests/session.test.ts @@ -0,0 +1,274 @@ +import { describe, expect, it, vi } from "vitest"; +import type { CuaFacadeTools } from "../src/executor.js"; +import { runGeminiCuaSession } from "../src/session.js"; +const logger = { log: () => {}, warn: () => {}, error: () => {} }; +describe("runGeminiCuaSession", () => { + it("sends computer use, acknowledges safety, returns screenshot and usage", async () => { + const requests: Record[] = []; + const client = { + generateContent: async (request: Record) => { + requests.push(request); + if (requests.length === 1) + return { + candidates: [ + { + content: { + parts: [ + { + functionCall: { + name: "click_at", + args: { x: 1, y: 2, safety_decision: "required" }, + }, + }, + ], + }, + }, + ], + usageMetadata: { + promptTokenCount: 10, + candidatesTokenCount: 4, + thoughtsTokenCount: 2, + cachedContentTokenCount: 3, + }, + }; + return { + candidates: [{ content: { parts: [{ text: "done" }] } }], + usageMetadata: { promptTokenCount: 5, candidatesTokenCount: 1 }, + }; + }, + }; + const result = await runGeminiCuaSession({ + prompt: "p", + model: "google/gemini-3.8-flash", + logger, + maxTurns: 3, + client, + tools: { execute: async () => ({ text: "ok" }) }, + facade: { + run: async () => "https://example.com", + screenshot: async () => ({ data: "PNG", mimeType: "image/png" }), + }, + }); + expect(result.status).toBe("completed"); + expect(result.tokenUsage).toMatchObject({ + input: 15, + output: 5, + reasoning: 2, + cached_input: 3, + }); + expect(requests[0]).toMatchObject({ + config: { + temperature: 1, + topP: 0.95, + topK: 40, + maxOutputTokens: 8192, + tools: [{ computerUse: { environment: "ENVIRONMENT_BROWSER" } }], + }, + }); + expect(JSON.stringify(requests[1])).toContain("safety_acknowledgement"); + const requestText = JSON.stringify(requests[1]); + expect(requestText).toContain('"output":"ok"'); + expect(requestText).toContain('"url":"https://example.com"'); + expect(requestText).toContain('"inlineData":{"mimeType":"image/png","data":"PNG"}'); + }); + + it("retries 429 once and does not retry 400", async () => { + let attempts = 0; + const client = { + generateContent: async () => { + attempts += 1; + if (attempts === 1) throw Object.assign(new Error("busy"), { status: 429 }); + return { candidates: [{ content: { parts: [{ text: "done" }] } }], usageMetadata: {} }; + }, + }; + const result = await runGeminiCuaSession({ + prompt: "p", + model: "gemini-3.8-flash", + logger, + maxTurns: 1, + client, + tools: { execute: async () => ({ text: "ok" }) }, + facade: { + run: async () => "", + screenshot: async () => ({ data: "", mimeType: "image/png" }), + }, + }); + expect(attempts).toBe(2); + expect(result.status).toBe("completed"); + let contractAttempts = 0; + const bad = { + generateContent: async () => { + contractAttempts += 1; + throw Object.assign(new Error("bad request"), { status: 400 }); + }, + }; + const failed = await runGeminiCuaSession({ + prompt: "p", + model: "gemini-3.8-flash", + logger, + maxTurns: 1, + client: bad, + tools: { execute: async () => ({ text: "ok" }) }, + facade: { + run: async () => "", + screenshot: async () => ({ data: "", mimeType: "image/png" }), + }, + }); + expect(contractAttempts).toBe(1); + expect(failed.status).toBe("sdk_error"); + }); + + it("returns max_turns and handles abort", async () => { + const facade: CuaFacadeTools = { + run: async () => "", + screenshot: async () => ({ data: "", mimeType: "image/png" }), + }; + const client = { + generateContent: async () => ({ + candidates: [{ content: { parts: [{ functionCall: { name: "wait", args: {} } }] } }], + usageMetadata: {}, + }), + }; + const result = await runGeminiCuaSession({ + prompt: "p", + model: "gemini-3.8-flash", + logger, + maxTurns: 1, + client, + tools: { execute: async () => ({ text: "ok" }) }, + facade, + }); + expect(result.status).toBe("max_turns"); + const controller = new AbortController(); + controller.abort(); + const aborted = await runGeminiCuaSession({ + prompt: "p", + model: "gemini-3.8-flash", + logger, + maxTurns: 1, + signal: controller.signal, + client, + tools: { execute: async () => ({ text: "ok" }) }, + facade, + }); + expect(aborted.status).toBe("sdk_error"); + }); + it("places system instructions and abort signal in SDK generation config", async () => { + const controller = new AbortController(); + let entered!: () => void; + const started = new Promise((resolve) => { + entered = resolve; + }); + const client = { + generateContent: async (request: Record) => { + expect(request).not.toHaveProperty("systemInstruction"); + const config = request.config as { systemInstruction: string; abortSignal: AbortSignal }; + expect(config.systemInstruction).toBe("native system policy"); + expect(config.abortSignal).toBe(controller.signal); + return new Promise((_, reject) => { + config.abortSignal.addEventListener("abort", () => reject(new Error("request aborted")), { + once: true, + }); + entered(); + }); + }, + }; + const pending = runGeminiCuaSession({ + prompt: "p", + model: "gemini-3.8-flash", + logger, + maxTurns: 2, + client, + signal: controller.signal, + systemPrompt: "native system policy", + tools: { execute: async () => ({ text: "ok" }) }, + facade: { + run: async () => "", + screenshot: async () => ({ data: "png", mimeType: "image/png" }), + }, + }); + await started; + controller.abort(); + expect((await pending).status).toBe("sdk_error"); + }); + + it("does not touch the browser or API when already aborted", async () => { + const controller = new AbortController(); + controller.abort(); + const run = vi.fn(async () => ""); + const generateContent = vi.fn(async () => ({})); + const result = await runGeminiCuaSession({ + prompt: "p", + model: "gemini-3.8-flash", + logger, + maxTurns: 1, + signal: controller.signal, + client: { generateContent }, + tools: { execute: async () => ({ text: "ok" }) }, + facade: { run, screenshot: async () => ({ data: "png", mimeType: "image/png" }) }, + }); + expect(result.status).toBe("sdk_error"); + expect(run).not.toHaveBeenCalled(); + expect(generateContent).not.toHaveBeenCalled(); + }); + + it("ends on terminal browser loss during observation without retry or later action", async () => { + const execute = vi.fn(async () => ({ text: "ok" })); + const screenshot = vi.fn(async () => { + throw new Error("Browser session lost (closed). Stop."); + }); + const generateContent = vi.fn(async () => ({ + candidates: [ + { + content: { + parts: [ + { functionCall: { name: "navigate", args: {} } }, + { functionCall: { name: "click_at", args: {} } }, + ], + }, + }, + ], + })); + const result = await runGeminiCuaSession({ + prompt: "p", + model: "gemini-3.8-flash", + logger, + maxTurns: 3, + client: { generateContent }, + tools: { execute }, + facade: { run: async () => "", screenshot }, + }); + expect(result.status).toBe("sdk_error"); + expect(result.stopReason).toContain("Browser session lost"); + expect(screenshot).toHaveBeenCalledOnce(); + expect(execute).toHaveBeenCalledOnce(); + expect(generateContent).toHaveBeenCalledOnce(); + }); + it.each([ + [undefined, false], + [{ promptTokenCount: 0, candidatesTokenCount: 0 }, true], + ] as const)( + "distinguishes absent usage from a reported zero (%j)", + async (usageMetadata, reported) => { + const result = await runGeminiCuaSession({ + prompt: "p", + model: "gemini-3.8-flash", + logger, + maxTurns: 1, + tools: { execute: async () => ({ text: "ok" }) }, + facade: { + run: async () => "", + screenshot: async () => ({ data: "png", mimeType: "image/png" }), + }, + client: { + generateContent: async () => ({ + candidates: [{ content: { parts: [{ text: "done" }] } }], + usageMetadata, + }), + }, + }); + expect(result.usageReported).toBe(reported); + expect(result.tokenUsage.total).toBe(0); + }, + ); +}); diff --git a/packages/integrations/gemini-cua-sdk/tsconfig.json b/packages/integrations/gemini-cua-sdk/tsconfig.json new file mode 100644 index 000000000..c11e8476b --- /dev/null +++ b/packages/integrations/gemini-cua-sdk/tsconfig.json @@ -0,0 +1,5 @@ +{ + "extends": "../../../tsconfig.json", + "compilerOptions": { "outDir": "dist" }, + "include": ["src", "tests"] +} diff --git a/packages/integrations/gemini-cua-sdk/tsdown.config.ts b/packages/integrations/gemini-cua-sdk/tsdown.config.ts new file mode 100644 index 000000000..81af4ee0f --- /dev/null +++ b/packages/integrations/gemini-cua-sdk/tsdown.config.ts @@ -0,0 +1,10 @@ +import { defineConfig } from "tsdown"; +export default defineConfig({ + entry: { index: "src/index.ts" }, + format: ["esm"], + platform: "node", + target: "node22", + dts: { sourcemap: true }, + sourcemap: true, + outDir: "dist", +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index bec0bee77..93e599650 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -256,6 +256,9 @@ catalogs: '@earendil-works/pi-coding-agent': specifier: 0.84.2 version: 0.84.2 + '@google/genai': + specifier: 1.52.0 + version: 1.52.0 '@mastra/core': specifier: ^1.55.0 version: 1.57.0 @@ -489,6 +492,9 @@ importers: '@browserbasehq/stagehand-integrations-fx-sdk': specifier: workspace:* version: link:../integrations/fx-sdk + '@browserbasehq/stagehand-integrations-gemini-cua-sdk': + specifier: workspace:* + version: link:../integrations/gemini-cua-sdk '@browserbasehq/stagehand-integrations-mastra-sdk': specifier: workspace:* version: link:../integrations/mastra-sdk @@ -908,6 +914,31 @@ importers: specifier: 'catalog:' version: 4.1.9(@opentelemetry/api@1.9.1)(@types/node@24.13.2)(vite@8.1.3(@types/node@24.13.2)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(yaml@2.9.0)) + packages/integrations/gemini-cua-sdk: + dependencies: + '@browserbasehq/stagehand': + specifier: workspace:* + version: link:../../sdk-ts + '@browserbasehq/stagehand-integrations': + specifier: workspace:* + version: link:../core + '@google/genai': + specifier: 'catalog:' + version: 1.52.0(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(bufferutil@4.1.0) + devDependencies: + '@types/node': + specifier: 'catalog:' + version: 24.13.2 + tsdown: + specifier: 'catalog:' + version: 0.22.3(publint@0.3.21)(tsx@4.23.1)(typescript@5.9.3) + typescript: + specifier: 'catalog:' + version: 5.9.3 + vitest: + specifier: 'catalog:' + version: 4.1.9(@opentelemetry/api@1.9.1)(@types/node@24.13.2)(vite@8.1.3(@types/node@24.13.2)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(yaml@2.9.0)) + packages/integrations/mastra: dependencies: '@ai-sdk/openai': diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index f4027e0bb..aff867dc5 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -5,6 +5,7 @@ packages: catalogMode: prefer catalog: + "@google/genai": 1.52.0 "@anthropic-ai/sdk": 0.93.0 "@cursor/sdk": 1.0.31 "@modelcontextprotocol/sdk": 1.29.0 diff --git a/turbo.json b/turbo.json index 84aaaf04d..92b1af47b 100644 --- a/turbo.json +++ b/turbo.json @@ -381,6 +381,24 @@ "**/*.test.ts", "$TURBO_ROOT$/vitest.config.ts" ] + }, + "@browserbasehq/stagehand-integrations-gemini-cua-sdk#build": { + "dependsOn": ["^build"], + "inputs": ["$TURBO_DEFAULT$", "!dist/**"], + "outputs": ["dist/**"] + }, + "@browserbasehq/stagehand-integrations-gemini-cua-sdk#typecheck": { + "dependsOn": ["^build"] + }, + "@browserbasehq/stagehand-integrations-gemini-cua-sdk#test:unit": { + "dependsOn": ["^build", "@browserbasehq/stagehand-integrations-gemini-cua-sdk#build"], + "inputs": [ + "$TURBO_DEFAULT$", + "tests/**", + "src/**", + "**/*.test.ts", + "$TURBO_ROOT$/vitest.config.ts" + ] } } } diff --git a/vitest.config.ts b/vitest.config.ts index db180ca4a..5f33f6f7a 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -13,6 +13,7 @@ export default defineConfig({ "packages/integrations/core/tests/**/*.test.ts", "packages/integrations/claude-agent-sdk/tests/**/*.test.ts", "packages/integrations/claude-cua-sdk/tests/**/*.test.ts", + "packages/integrations/gemini-cua-sdk/tests/**/*.test.ts", "packages/integrations/codex-sdk/tests/**/*.test.ts", "packages/integrations/mastra-sdk/tests/**/*.test.ts", "packages/integrations/pi-sdk/tests/**/*.test.ts", From 360c780b81a78af33e37402db3576a4d47f5419a Mon Sep 17 00:00:00 2001 From: miguel Date: Mon, 7 Sep 2026 17:50:36 -0700 Subject: [PATCH 2/5] Reject incomplete and blocked Gemini CUA completions --- .../gemini-cua-sdk/src/session.ts | 8 ++++ .../gemini-cua-sdk/tests/session.test.ts | 48 ++++++++++++++++++- 2 files changed, 55 insertions(+), 1 deletion(-) diff --git a/packages/integrations/gemini-cua-sdk/src/session.ts b/packages/integrations/gemini-cua-sdk/src/session.ts index eb94d91be..9de4eac4b 100644 --- a/packages/integrations/gemini-cua-sdk/src/session.ts +++ b/packages/integrations/gemini-cua-sdk/src/session.ts @@ -148,12 +148,20 @@ export async function runGeminiCuaSession( .join("\n") .trim(); events.push({ type: "assistant", turn: turns, text, parts, usage: turnUsage }); + if (!candidate) throw new Error("Gemini returned no response candidate."); + // A successful HTTP request can still be blocked or truncated. Retain + // its evidence and usage, but do not execute partial calls or report it + // as a completed task. + if (candidate.finishReason && candidate.finishReason !== "STOP") { + throw new Error(`Gemini response ended with ${candidate.finishReason}.`); + } contents.push({ role: "model", parts }); const calls = parts .filter(isRecord) .map((part) => part.functionCall) .filter(isRecord); if (calls.length === 0) { + if (!text) throw new Error("Gemini returned neither tool calls nor an answer."); finalMessage = text; status = "completed"; stopReason = candidate?.finishReason; diff --git a/packages/integrations/gemini-cua-sdk/tests/session.test.ts b/packages/integrations/gemini-cua-sdk/tests/session.test.ts index a10c8a82f..bd8c7c8a0 100644 --- a/packages/integrations/gemini-cua-sdk/tests/session.test.ts +++ b/packages/integrations/gemini-cua-sdk/tests/session.test.ts @@ -3,6 +3,52 @@ import type { CuaFacadeTools } from "../src/executor.js"; import { runGeminiCuaSession } from "../src/session.js"; const logger = { log: () => {}, warn: () => {}, error: () => {} }; describe("runGeminiCuaSession", () => { + it.each([ + ["MAX_TOKENS", { finishReason: "MAX_TOKENS", content: { parts: [{ text: "incomplete" }] } }], + ["SAFETY", { finishReason: "SAFETY", content: { parts: [] } }], + [ + "MALFORMED_FUNCTION_CALL", + { finishReason: "MALFORMED_FUNCTION_CALL", content: { parts: [] } }, + ], + ["no response candidate", undefined], + ["neither tool calls nor an answer", { finishReason: "STOP", content: { parts: [] } }], + [ + "MAX_TOKENS", + { + finishReason: "MAX_TOKENS", + content: { parts: [{ functionCall: { name: "click", args: { x: 1, y: 2 } } }] }, + }, + ], + ])("does not complete or act on an unusable response (%s)", async (reason, candidate) => { + const execute = vi.fn(async () => ({ text: "ok" })); + const generateContent = vi.fn(async () => ({ + candidates: candidate ? [candidate] : [], + usageMetadata: { promptTokenCount: 7, candidatesTokenCount: 3 }, + })); + const result = await runGeminiCuaSession({ + prompt: "p", + model: "gemini-3.8-flash", + logger, + maxTurns: 3, + client: { generateContent }, + tools: { execute }, + facade: { + run: async () => "about:blank", + screenshot: async () => { + throw new Error("unexpected screenshot"); + }, + }, + }); + expect(result.status).toBe("sdk_error"); + expect(result.stopReason).toContain(reason); + expect(result.finalMessage).toBe(""); + expect(result.iterationError).toBeInstanceOf(Error); + expect(result.tokenUsage).toMatchObject({ input: 7, output: 3 }); + expect(result.events).toHaveLength(1); + expect(execute).not.toHaveBeenCalled(); + expect(generateContent).toHaveBeenCalledOnce(); + }); + it("sends computer use, acknowledges safety, returns screenshot and usage", async () => { const requests: Record[] = []; const client = { @@ -32,7 +78,7 @@ describe("runGeminiCuaSession", () => { }, }; return { - candidates: [{ content: { parts: [{ text: "done" }] } }], + candidates: [{ finishReason: "STOP", content: { parts: [{ text: "done" }] } }], usageMetadata: { promptTokenCount: 5, candidatesTokenCount: 1 }, }; }, From 4f3b9dd08b6e1a1b6d8fea1836c54f43595ed730 Mon Sep 17 00:00:00 2001 From: miguel Date: Mon, 7 Sep 2026 17:50:36 -0700 Subject: [PATCH 3/5] Honor Gemini CUA gestures and action arguments through the SDK --- .../gemini-cua-sdk/ARCHITECTURE.md | 4 + .../gemini-cua-sdk/src/executor.ts | 38 +++--- .../gemini-cua-sdk/tests/executor.test.ts | 14 +-- .../gemini-cua-sdk/tests/sdk-boundary.test.ts | 110 ++++++++++++++++++ 4 files changed, 140 insertions(+), 26 deletions(-) create mode 100644 packages/integrations/gemini-cua-sdk/tests/sdk-boundary.test.ts diff --git a/packages/integrations/gemini-cua-sdk/ARCHITECTURE.md b/packages/integrations/gemini-cua-sdk/ARCHITECTURE.md index 2c48bfd09..89c678602 100644 --- a/packages/integrations/gemini-cua-sdk/ARCHITECTURE.md +++ b/packages/integrations/gemini-cua-sdk/ARCHITECTURE.md @@ -13,3 +13,7 @@ The common evaluation policy is sent once via `config.systemInstruction`, the in Budgets count API turns, which can contain multiple tool calls. The exact per-action response and image sent to the model are retained on the tool-result event and attached by tool-use ID, so failed calls do not shift observations. The eval adapter uses these returned images rather than capturing another screenshot for each action. Historical index-based observations remain readable by the trajectory adapter. Input/cache/output/reasoning counters retain the provider's separate fields for shared usage normalization. Coordinate actions use a 1288×711 viewport and Gemini's normalized coordinates. The executor supports the provider actions listed in its switch; unknown actions return explicit errors. It does not add facade reconnection or alternate browser ownership. + +Browser gestures use the existing SDK Page primitives through the common facade. Drag uses `dragAndDrop` with two movement steps, preserving the midpoint before the endpoint. Gemini `hotkey` arrays form a single key chord; `wait` honors its seconds argument (default 1), while the legacy `wait_5_seconds` remains 5 seconds. The current `scroll` action defaults to 300 pixels, and legacy `scroll_at` keeps its 800-pixel default. + +Blocked, truncated, malformed, missing, and empty terminal responses are recorded as SDK errors with their response events and usage preserved. No action from a truncated response is executed, and incomplete text is not reported as a completed task. diff --git a/packages/integrations/gemini-cua-sdk/src/executor.ts b/packages/integrations/gemini-cua-sdk/src/executor.ts index 23ab30e02..4dd9b86ce 100644 --- a/packages/integrations/gemini-cua-sdk/src/executor.ts +++ b/packages/integrations/gemini-cua-sdk/src/executor.ts @@ -103,7 +103,7 @@ export class GeminiCuaExecutor implements GeminiToolExecutor { case "press_key": case "press_keys": case "hotkey": - return await this.pressKeys(input); + return await this.pressKeys(name, input); case "scroll_document": case "scroll_at": case "scroll": @@ -117,9 +117,17 @@ export class GeminiCuaExecutor implements GeminiToolExecutor { await this.runWithTabs("await page.goForward();"); return { text: "Went forward." }; case "wait_5_seconds": - case "wait": - await this.runWithTabs("await page.waitForTimeout(5000);"); - return { text: "Waited 5 seconds." }; + case "wait": { + const seconds = name === "wait_5_seconds" ? 5 : (input.seconds ?? 1); + if (typeof seconds !== "number" || seconds < 0 || !Number.isFinite(seconds * 1000)) { + return { + text: "Error: wait requires a finite nonnegative duration in seconds.", + isError: true, + }; + } + await this.runWithTabs(`await page.waitForTimeout(${seconds * 1000});`); + return { text: `Waited ${seconds} seconds.` }; + } case "take_screenshot": case "screenshot": return { text: "Screenshot captured." }; @@ -186,7 +194,7 @@ export class GeminiCuaExecutor implements GeminiToolExecutor { return { text: `Typed ${JSON.stringify(String(input.text ?? ""))}.` }; } - private async pressKeys(input: Record): Promise { + private async pressKeys(name: string, input: Record): Promise { const raw = input.keys ?? input.key ?? input.text; const values = Array.isArray(raw) ? raw @@ -194,7 +202,12 @@ export class GeminiCuaExecutor implements GeminiToolExecutor { .split(/\s+/u) .filter(Boolean); if (values.length === 0) return { text: "Error: no keys supplied.", isError: true }; - const pressed = values.map((value) => this.chord(value)); + // Gemini hotkey arrays describe one simultaneous chord, while the legacy + // key-sequence aliases still execute their entries one after another. + const pressed = + name === "hotkey" && Array.isArray(raw) + ? [values.map((value) => this.chord(value)).join("+")] + : values.map((value) => this.chord(value)); await this.runWithTabs( pressed .map((key) => `await batchStagehand.page.keyPress(${JSON.stringify(key)});`) @@ -217,7 +230,9 @@ export class GeminiCuaExecutor implements GeminiToolExecutor { ? input.magnitude : typeof input.magnitude_in_pixels === "number" ? input.magnitude_in_pixels - : 800; + : name === "scroll" + ? 300 + : 800; const deltaX = direction === "left" ? -amount : direction === "right" ? amount : 0; const deltaY = direction === "up" ? -amount : direction === "down" ? amount : 0; await this.runWithTabs(`await batchStagehand.page.scroll(${x}, ${y}, ${deltaX}, ${deltaY});`); @@ -248,13 +263,10 @@ export class GeminiCuaExecutor implements GeminiToolExecutor { denormalizeCoordinate(values[1], HEIGHT), ]; const end = [denormalizeCoordinate(values[2], WIDTH), denormalizeCoordinate(values[3], HEIGHT)]; + // Delegate the full gesture to the SDK, retaining the midpoint movement + // before the destination without relying on a raw CDP method on Page. await this.runWithTabs( - `const __p = batchStagehand.page; -await __p.sendCDP("Input.dispatchMouseEvent", { type: "mouseMoved", x: ${start[0]}, y: ${start[1]}, button: "none" }); -await __p.sendCDP("Input.dispatchMouseEvent", { type: "mousePressed", x: ${start[0]}, y: ${start[1]}, button: "left", clickCount: 1 }); -await __p.sendCDP("Input.dispatchMouseEvent", { type: "mouseMoved", x: ${(start[0] + end[0]) / 2}, y: ${(start[1] + end[1]) / 2}, button: "left" }); -await __p.sendCDP("Input.dispatchMouseEvent", { type: "mouseMoved", x: ${end[0]}, y: ${end[1]}, button: "left" }); -await __p.sendCDP("Input.dispatchMouseEvent", { type: "mouseReleased", x: ${end[0]}, y: ${end[1]}, button: "left", clickCount: 1 });`, + `await batchStagehand.page.dragAndDrop(${start[0]}, ${start[1]}, ${end[0]}, ${end[1]}, { steps: 2 });`, ); return { text: `Dragged from (${start[0]}, ${start[1]}) to (${end[0]}, ${end[1]}).` }; } diff --git a/packages/integrations/gemini-cua-sdk/tests/executor.test.ts b/packages/integrations/gemini-cua-sdk/tests/executor.test.ts index 87b445ba9..010f412a4 100644 --- a/packages/integrations/gemini-cua-sdk/tests/executor.test.ts +++ b/packages/integrations/gemini-cua-sdk/tests/executor.test.ts @@ -54,19 +54,7 @@ describe("GeminiCuaExecutor", () => { expect(calls[1]).toContain('page.keyPress("PageUp")'); expect(calls[2]).toContain('page.goto("https://www.google.com")'); expect(calls[3]).toContain( - 'await __p.sendCDP("Input.dispatchMouseEvent", { type: "mouseMoved", x: 0, y: 0, button: "none" });', - ); - expect(calls[3]).toContain( - 'await __p.sendCDP("Input.dispatchMouseEvent", { type: "mousePressed", x: 0, y: 0, button: "left", clickCount: 1 });', - ); - expect(calls[3]).toContain( - 'await __p.sendCDP("Input.dispatchMouseEvent", { type: "mouseMoved", x: 643, y: 355, button: "left" });', - ); - expect(calls[3]).toContain( - 'await __p.sendCDP("Input.dispatchMouseEvent", { type: "mouseMoved", x: 1286, y: 710, button: "left" });', - ); - expect(calls[3]).toContain( - 'await __p.sendCDP("Input.dispatchMouseEvent", { type: "mouseReleased", x: 1286, y: 710, button: "left", clickCount: 1 });', + "await batchStagehand.page.dragAndDrop(0, 0, 1286, 710, { steps: 2 });", ); }); diff --git a/packages/integrations/gemini-cua-sdk/tests/sdk-boundary.test.ts b/packages/integrations/gemini-cua-sdk/tests/sdk-boundary.test.ts new file mode 100644 index 000000000..05fde9ebe --- /dev/null +++ b/packages/integrations/gemini-cua-sdk/tests/sdk-boundary.test.ts @@ -0,0 +1,110 @@ +import { Page } from "@browserbasehq/stagehand"; +import { describe, expect, it } from "vitest"; +import { GeminiCuaExecutor } from "../src/executor.js"; + +const logger = { log() {}, warn() {}, error() {} }; + +function sdkFacade() { + const calls: Array<{ method: string; params: unknown }> = []; + const page = new Page( + { + send: async (method, params) => { + calls.push({ method: method.name, params: method.params.parse(params) }); + return { ok: true } as never; + }, + onNotification: () => () => {}, + }, + { pageId: "p1" }, + ); + const waits: number[] = []; + const AsyncFunction = Object.getPrototypeOf(async function () {}).constructor; + const executor = new GeminiCuaExecutor( + { + run: async (code) => + new AsyncFunction("batchStagehand", "page", code)( + { page }, + { + waitForTimeout: async (ms: number) => { + waits.push(ms); + }, + }, + ), + screenshot: async () => { + throw new Error("unexpected screenshot"); + }, + }, + logger, + ); + return { executor, calls, waits }; +} + +describe("Gemini CUA SDK Page boundary", () => { + it.each([ + { x: 0, y: 0, destination_x: 999, destination_y: 999 }, + { start_x: 0, start_y: 0, end_x: 999, end_y: 999 }, + ])("executes normalized drags through the actual SDK Page (%j)", async (input) => { + const { executor, calls } = sdkFacade(); + const result = await executor.execute("drag_and_drop", input); + expect(result.isError, result.text).not.toBe(true); + expect(calls).toEqual([ + { + method: "page.drag_and_drop", + params: { pageId: "p1", fromX: 0, fromY: 0, toX: 1286, toY: 710, options: { steps: 2 } }, + }, + ]); + }); + it("executes a hotkey array as one native key chord", async () => { + const { executor, calls } = sdkFacade(); + const result = await executor.execute("hotkey", { keys: ["Control", "a"] }); + expect(result.isError, result.text).not.toBe(true); + expect(calls).toEqual([ + { method: "page.key_press", params: { pageId: "p1", key: "Control+a" } }, + ]); + }); + + it.each([ + ["wait", {}, 1000], + ["wait", { seconds: 2.5 }, 2500], + ["wait", { seconds: 0 }, 0], + ["wait_5_seconds", {}, 5000], + ])("honors %s duration %j", async (name, input, ms) => { + const { executor, waits } = sdkFacade(); + const result = await executor.execute(name, input); + expect(result.isError, result.text).not.toBe(true); + expect(waits).toEqual([ms]); + }); + + it.each([ + ["scroll", {}, 300], + ["scroll_at", {}, 800], + ["scroll", { magnitude_in_pixels: 125 }, 125], + ["scroll_at", { magnitude: 90 }, 90], + ])("preserves the %s scroll amount %j", async (name, magnitude, amount) => { + const { executor, calls } = sdkFacade(); + const result = await executor.execute(name, { + x: 500, + y: 500, + direction: "down", + ...magnitude, + }); + expect(result.isError, result.text).not.toBe(true); + expect(calls).toEqual([ + { + method: "page.scroll", + params: { pageId: "p1", x: 644, y: 355, deltaX: 0, deltaY: amount }, + }, + ]); + }); + + it.each([ + ["hotkey", { keys: [] }], + ["wait", { seconds: -1 }], + ["wait", { seconds: "2" }], + ["wait", { seconds: Number.POSITIVE_INFINITY }], + ])("rejects invalid %s input before execution", async (name, input) => { + const { executor, calls, waits } = sdkFacade(); + expect((await executor.execute(name, input)).isError).toBe(true); + expect(calls).toEqual([]); + expect(waits).toEqual([]); + }); +}); From 628eff28a89c2a8db866b959357609dac50eee6c Mon Sep 17 00:00:00 2001 From: miguel Date: Mon, 7 Sep 2026 18:01:32 -0700 Subject: [PATCH 4/5] Wire trusted session-loss state into Gemini CUA --- packages/evals/framework/geminiCuaToolAdapter.ts | 2 +- .../evals/tests/framework/geminiCuaMount.test.ts | 14 ++++++++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/packages/evals/framework/geminiCuaToolAdapter.ts b/packages/evals/framework/geminiCuaToolAdapter.ts index 9d69a684c..ed62e8cd2 100644 --- a/packages/evals/framework/geminiCuaToolAdapter.ts +++ b/packages/evals/framework/geminiCuaToolAdapter.ts @@ -59,7 +59,7 @@ export async function prepareGeminiCuaToolAdapter(input: { const mount = runtime.running.agentMount; if (!callTool || !mount) throw new EvalsError(`Tool surface "${toolSurface}" does not expose runner-side tool calls.`); - const facade = bridgeCuaFacadeTools(callTool); + const facade = bridgeCuaFacadeTools(callTool, undefined, runtime.running.browserSessionLoss); const executor = new GeminiCuaExecutor(facade, input.logger); return { toolSurface, diff --git a/packages/evals/tests/framework/geminiCuaMount.test.ts b/packages/evals/tests/framework/geminiCuaMount.test.ts index e593cf43e..23a85b54a 100644 --- a/packages/evals/tests/framework/geminiCuaMount.test.ts +++ b/packages/evals/tests/framework/geminiCuaMount.test.ts @@ -25,9 +25,10 @@ const input = { describe("Gemini native mount ownership", () => { it("uses the supplied browser and typed bridge, preserves loss metadata, and cleans up once", async () => { const cleanup = vi.fn(async () => {}); - const loss = (): undefined => undefined; + let currentLoss: { cause: string } | undefined; + const loss = () => currentLoss; const callTool = vi.fn(async () => ({ - content: [{ type: "text", text: "{}" }], + content: [{ type: "text", text: '{"type":"undefined"}' }], })); const browserSession = { provider: "local" as const, sessionId: "owned-fixture" }; vi.mocked(startAgentToolRuntime).mockResolvedValue({ @@ -64,6 +65,15 @@ describe("Gemini native mount ownership", () => { { code: expect.stringContaining('page.goto("https://fixture.test"') }, { timeoutMs: 90_000 }, ); + currentLoss = { cause: "closed" }; + await expect( + adapter.executor.execute( + "navigate", + { url: "https://fixture.test" }, + { toolUseId: "after-loss" }, + ), + ).rejects.toThrow("Browser session lost (confirmed by eval runner)"); + expect(callTool).toHaveBeenCalledOnce(); await Promise.all([adapter.cleanup(), adapter.cleanup()]); expect(cleanup).toHaveBeenCalledOnce(); }); From a48e044ecc0cde531dc2a9673d9a2ed9f980fa7f Mon Sep 17 00:00:00 2001 From: miguel Date: Mon, 7 Sep 2026 18:54:39 -0700 Subject: [PATCH 5/5] fix(gemini-cua): validate responses and preserve trusted execution evidence --- packages/evals/README.md | 2 +- packages/evals/framework/geminiCuaRunner.ts | 13 +- .../evals/framework/geminiCuaToolAdapter.ts | 7 +- .../framework/harnesses/geminiCuaAdapter.ts | 24 ++- .../tests/framework/geminiCuaAdapter.test.ts | 53 ++++++ .../tests/framework/geminiCuaMount.test.ts | 2 +- .../tests/framework/geminiCuaRunner.test.ts | 16 +- .../gemini-cua-sdk/ARCHITECTURE.md | 6 +- .../integrations/gemini-cua-sdk/src/errors.ts | 11 ++ .../gemini-cua-sdk/src/executor.ts | 27 ++- .../gemini-cua-sdk/src/session.ts | 69 ++++--- .../gemini-cua-sdk/src/transcript.ts | 7 +- .../gemini-cua-sdk/tests/executor.test.ts | 56 +++++- .../gemini-cua-sdk/tests/session.test.ts | 171 +++++++++++++++++- .../gemini-cua-sdk/tests/transcript.test.ts | 23 +++ 15 files changed, 423 insertions(+), 64 deletions(-) create mode 100644 packages/integrations/gemini-cua-sdk/src/errors.ts create mode 100644 packages/integrations/gemini-cua-sdk/tests/transcript.test.ts diff --git a/packages/evals/README.md b/packages/evals/README.md index 038bd6356..1456a9667 100644 --- a/packages/evals/README.md +++ b/packages/evals/README.md @@ -88,7 +88,7 @@ A live run paints an in-place progress table, then prints a final summary with a See [the harness contract](docs/harness-contract.md) for tool surfaces, prompt policy, budget units, session diagnostics, verification, and usage accounting. -See [HardBench](datasets/hardbenchmark/MANIFEST.md) for corpus selection and rubric v1.2. +See [HardBench](datasets/hardbenchmark/README.md) for corpus selection and rubric v1.2. ## Adding a bench task diff --git a/packages/evals/framework/geminiCuaRunner.ts b/packages/evals/framework/geminiCuaRunner.ts index 9dec24feb..70e88e34c 100644 --- a/packages/evals/framework/geminiCuaRunner.ts +++ b/packages/evals/framework/geminiCuaRunner.ts @@ -1,6 +1,7 @@ import { buildGeminiCuaTranscript, runGeminiCuaSession, + normalizeGeminiCuaModel, type GeminiGenerateClient, } from "@browserbasehq/stagehand-integrations-gemini-cua-sdk"; import type { AvailableModel } from "stagehand-v3"; @@ -38,15 +39,16 @@ export function buildGeminiCuaPrompt( return buildExternalHarnessPrompt({ plan, toolInstructions, resultContract: "marker" }); } export function assertGeminiCuaModel(model: string): void { - const bare = model.includes("/") ? model.slice(model.indexOf("/") + 1) : model; - const provider = model.includes("/") ? model.slice(0, model.indexOf("/")) : undefined; - if ((provider !== undefined && provider !== "google") || !bare.startsWith("gemini-")) - throw new EvalsError(`gemini_cua runs Google Gemini models only (received "${model}").`); + try { + normalizeGeminiCuaModel(model); + } catch { + throw new EvalsError("gemini_cua requires a non-empty model identifier."); + } } export async function runGeminiCuaAgent(input: GeminiCuaRunnerInput): Promise { assertGeminiCuaModel(input.model); const maxTurns = resolveStepBudget({ - harnessEnvKey: "EVAL_GEMINI_CUA_MAX_STEPS", + harnessEnvKey: "EVAL_GEMINI_CUA_MAX_TURNS", dataset: input.plan.dataset, harnessDefault: 50, }); @@ -73,6 +75,7 @@ export async function runGeminiCuaAgent(input: GeminiCuaRunnerInput): Promise(); let pendingReasoning = ""; let finalAnswer = result.finalAnswer; + let lastImage: Buffer | undefined; for (const event of result.events) { if (event.type === "assistant") { pendingReasoning = event.text; - if (!result.finalAnswer && event.text) finalAnswer = event.text; + if (result.finalAnswer === undefined && event.text) finalAnswer = event.text; } else if (event.type === "tool_use") { const call: NormalizedToolCall = { name: event.name, args: event.input, result: undefined, - ok: true, + ok: false, + error: "Tool call did not return a result.", reasoning: pendingReasoning.trim() || undefined, probeEvidence: result.stepObservationsByToolUse?.get(event.id) ?? @@ -48,22 +50,28 @@ export class GeminiCuaTrajectoryAdapter implements TrajectoryAdapter { + const trajectory = geminiCuaAdapter.fromHarnessResult( + { + finalAnswer: "", + status: "error", + events: [ + { + type: "assistant", + turn: 1, + text: "I will finish now", + parts: [], + usage: { input: 0, output: 0, reasoning: 0, cached_input: 0, total: 0 }, + }, + { type: "tool_use", turn: 1, id: "unfinished", name: "click", input: {} }, + ], + }, + { id: "incomplete", instruction: "act" }, + ); + expect(trajectory.finalAnswer).toBe(""); + expect(trajectory.steps[0]?.toolOutput).toMatchObject({ ok: false }); +}); + +it("uses the last recorded tool screenshot without losing final metadata or replacing a final capture", () => { + const shot = Buffer.from("last screenshot"); + const events: GeminiCuaSessionEvent[] = [ + { type: "tool_use", turn: 1, id: "one", name: "click", input: {} }, + { + type: "tool_result", + turn: 1, + id: "one", + name: "click", + text: "ok", + error: false, + image: { data: shot.toString("base64"), mimeType: "image/png" }, + }, + ]; + const result = { + events, + finalObservation: { url: "https://fixture.test", ariaTree: "button Done" }, + }; + const fallback = geminiCuaAdapter.fromHarnessResult(result, { + id: "last-image", + instruction: "act", + }); + expect(fallback.finalObservation).toMatchObject({ ...result.finalObservation, screenshot: shot }); + const final = Buffer.from("terminal capture"); + const captured = geminiCuaAdapter.fromHarnessResult( + { ...result, finalObservation: { ...result.finalObservation, screenshot: final } }, + { id: "final", instruction: "act" }, + ); + expect(captured.finalObservation?.screenshot).toEqual(final); +}); diff --git a/packages/evals/tests/framework/geminiCuaMount.test.ts b/packages/evals/tests/framework/geminiCuaMount.test.ts index 23a85b54a..4051a6855 100644 --- a/packages/evals/tests/framework/geminiCuaMount.test.ts +++ b/packages/evals/tests/framework/geminiCuaMount.test.ts @@ -72,7 +72,7 @@ describe("Gemini native mount ownership", () => { { url: "https://fixture.test" }, { toolUseId: "after-loss" }, ), - ).rejects.toThrow("Browser session lost (confirmed by eval runner)"); + ).rejects.toThrow("Browser session lost"); expect(callTool).toHaveBeenCalledOnce(); await Promise.all([adapter.cleanup(), adapter.cleanup()]); expect(cleanup).toHaveBeenCalledOnce(); diff --git a/packages/evals/tests/framework/geminiCuaRunner.test.ts b/packages/evals/tests/framework/geminiCuaRunner.test.ts index 3aa32a426..d680de277 100644 --- a/packages/evals/tests/framework/geminiCuaRunner.test.ts +++ b/packages/evals/tests/framework/geminiCuaRunner.test.ts @@ -1,5 +1,7 @@ import { GEMINI_CUA_TOOL_INSTRUCTIONS } from "../../framework/geminiCuaToolAdapter.js"; -import { describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { EvalsError } from "../../errors.js"; +afterEach(() => vi.unstubAllEnvs()); import type { GeminiGenerateClient } from "@browserbasehq/stagehand-integrations-gemini-cua-sdk"; import type { AvailableModel } from "stagehand-v3"; import { runGeminiCuaAgent, assertGeminiCuaModel } from "../../framework/geminiCuaRunner.js"; @@ -8,6 +10,8 @@ import { EvalLogger } from "../../logger.js"; describe("Gemini CUA eval system prompt", () => { it("sends the common policy once through the native system channel", async () => { + vi.stubEnv("EVAL_GEMINI_CUA_MAX_TURNS", "3"); + vi.stubEnv("AGENT_EVAL_MAX_STEPS", "9"); let request: Record | undefined; const client: GeminiGenerateClient = { generateContent: async (params) => { @@ -54,6 +58,7 @@ describe("Gemini CUA eval system prompt", () => { }); expect(result._success).toBe(true); + expect(result).toMatchObject({ metrics: { step_budget: { value: 3 } } }); expect( String((request?.config as { systemInstruction?: string })?.systemInstruction).split( EVAL_SYSTEM_PROMPT, @@ -75,9 +80,10 @@ describe("Gemini CUA eval system prompt", () => { }); }); -it("rejects models from a different provider", () => { +it("lets the native provider validate model identifiers", () => { expect(() => assertGeminiCuaModel("google/gemini-3.8-flash")).not.toThrow(); - expect(() => assertGeminiCuaModel("anthropic/gemini-3.8-flash")).toThrow( - "Google Gemini models only", - ); + expect(() => assertGeminiCuaModel("future-model")).not.toThrow(); + expect(() => assertGeminiCuaModel("projects/p/models/custom")).not.toThrow(); + expect(() => assertGeminiCuaModel("other-provider/custom")).not.toThrow(); + expect(() => assertGeminiCuaModel("google/")).toThrow(EvalsError); }); diff --git a/packages/integrations/gemini-cua-sdk/ARCHITECTURE.md b/packages/integrations/gemini-cua-sdk/ARCHITECTURE.md index 89c678602..e9a2b61a1 100644 --- a/packages/integrations/gemini-cua-sdk/ARCHITECTURE.md +++ b/packages/integrations/gemini-cua-sdk/ARCHITECTURE.md @@ -10,10 +10,14 @@ The evals `gemini_cua` harness mounts only `google_computer_use`; it uses the sa The common evaluation policy is sent once via `config.systemInstruction`, the installed Google SDK request field. `config.abortSignal` cancels the active client request; aborts also stop retry waits and are checked before browser initialization and between actions. A terminal browser loss ends the session without retrying screenshots. Transient observation failures retain the existing one-retry behavior. Eval cleanup is bounded and idempotent. -Budgets count API turns, which can contain multiple tool calls. The exact per-action response and image sent to the model are retained on the tool-result event and attached by tool-use ID, so failed calls do not shift observations. The eval adapter uses these returned images rather than capturing another screenshot for each action. Historical index-based observations remain readable by the trajectory adapter. Input/cache/output/reasoning counters retain the provider's separate fields for shared usage normalization. +Budgets count API turns, which can contain multiple tool calls. `EVAL_GEMINI_CUA_MAX_TURNS` overrides the shared or dataset budget. The exact per-action response and image sent to the model are retained on the tool-result event and attached by tool-use ID, so failed calls do not shift observations. The eval adapter uses these returned images rather than capturing another screenshot for each action. Historical index-based observations remain readable by the trajectory adapter. Input/cache/output/reasoning counters retain the provider's separate fields for shared usage normalization. Coordinate actions use a 1288×711 viewport and Gemini's normalized coordinates. The executor supports the provider actions listed in its switch; unknown actions return explicit errors. It does not add facade reconnection or alternate browser ownership. Browser gestures use the existing SDK Page primitives through the common facade. Drag uses `dragAndDrop` with two movement steps, preserving the midpoint before the endpoint. Gemini `hotkey` arrays form a single key chord; `wait` honors its seconds argument (default 1), while the legacy `wait_5_seconds` remains 5 seconds. The current `scroll` action defaults to 300 pixels, and legacy `scroll_at` keeps its 800-pixel default. Blocked, truncated, malformed, missing, and empty terminal responses are recorded as SDK errors with their response events and usage preserved. No action from a truncated response is executed, and incomplete text is not reported as a completed task. + +Terminal loss is determined by the core facade error class or the runner-owned loss getter, never a page or agent error string. Malformed function-call batches fail before any call executes. Model identifiers are forwarded to the native provider for validation, stripping only the optional `google/` catalog alias. Session failures use sanitized `HarnessAdapterError` values. + +An explicit empty final answer remains empty, and a tool call without a result is unsuccessful. If terminal evidence has no screenshot, the last model-visible tool image is retained alongside available final metadata; it represents the last captured frame, not a new terminal capture. diff --git a/packages/integrations/gemini-cua-sdk/src/errors.ts b/packages/integrations/gemini-cua-sdk/src/errors.ts new file mode 100644 index 000000000..e145784b6 --- /dev/null +++ b/packages/integrations/gemini-cua-sdk/src/errors.ts @@ -0,0 +1,11 @@ +import { StagehandFacadeSessionLostError } from "@browserbasehq/stagehand-integrations/facade"; + +export type BrowserSessionLossReader = () => { cause: string } | undefined; + +/** Transport ownership, never page/model error text, decides terminal loss. */ +export function isTerminalFacadeError( + error: unknown, + browserSessionLoss?: BrowserSessionLossReader, +): boolean { + return error instanceof StagehandFacadeSessionLostError || browserSessionLoss?.() !== undefined; +} diff --git a/packages/integrations/gemini-cua-sdk/src/executor.ts b/packages/integrations/gemini-cua-sdk/src/executor.ts index 4dd9b86ce..c9f413911 100644 --- a/packages/integrations/gemini-cua-sdk/src/executor.ts +++ b/packages/integrations/gemini-cua-sdk/src/executor.ts @@ -1,8 +1,10 @@ +import type { StagehandFacadeTools } from "@browserbasehq/stagehand-integrations/facade"; import { - isBrowserSessionLostError, - type StagehandFacadeTools, -} from "@browserbasehq/stagehand-integrations/facade"; -import type { HarnessLogger } from "@browserbasehq/stagehand-integrations/harness"; + HarnessAdapterError, + sanitizeErrorMessage, + type HarnessLogger, +} from "@browserbasehq/stagehand-integrations/harness"; +import { isTerminalFacadeError, type BrowserSessionLossReader } from "./errors.js"; export type CuaFacadeTools = Pick; @@ -51,6 +53,7 @@ export class GeminiCuaExecutor implements GeminiToolExecutor { private readonly tools: CuaFacadeTools, private readonly logger: HarnessLogger, private readonly onMutation?: (toolUseId: string) => Promise, + private readonly browserSessionLoss?: BrowserSessionLossReader, ) {} async execute( @@ -61,8 +64,7 @@ export class GeminiCuaExecutor implements GeminiToolExecutor { const result = await this.executeAction(name, input, context); if (!result.isError && context.toolUseId && this.onMutation) { await this.onMutation(context.toolUseId).catch((error: unknown) => { - if (isBrowserSessionLostError(error instanceof Error ? error.message : String(error))) - throw error; + if (isTerminalFacadeError(error, this.browserSessionLoss)) throw error; }); } return result; @@ -74,7 +76,7 @@ export class GeminiCuaExecutor implements GeminiToolExecutor { context: { signal?: AbortSignal } = {}, ): Promise { try { - if (context.signal?.aborted) throw new Error("operation aborted"); + if (context.signal?.aborted) throw new HarnessAdapterError("operation aborted"); switch (name) { case "click_at": case "click": @@ -143,12 +145,9 @@ export class GeminiCuaExecutor implements GeminiToolExecutor { return { text: `Error: unknown computer-use action "${name}".`, isError: true }; } } catch (error) { - if ( - context.signal?.aborted || - isBrowserSessionLostError(error instanceof Error ? error.message : String(error)) - ) + if (context.signal?.aborted || isTerminalFacadeError(error, this.browserSessionLoss)) throw error; - const message = error instanceof Error ? error.message : String(error); + const message = sanitizeErrorMessage(error instanceof Error ? error.message : String(error)); this.logger.warn({ category: "gemini_cua", message: `${name} failed: ${message}`, level: 1 }); return { text: `Error: ${message}`, isError: true }; } @@ -257,7 +256,7 @@ export class GeminiCuaExecutor implements GeminiToolExecutor { input.end_y ?? input.destination_y, ]; if (!values.every((value) => typeof value === "number" && Number.isFinite(value))) - throw new Error("drag requires four finite coordinates"); + throw new HarnessAdapterError("drag requires four finite coordinates"); const start = [ denormalizeCoordinate(values[0], WIDTH), denormalizeCoordinate(values[1], HEIGHT), @@ -278,7 +277,7 @@ export class GeminiCuaExecutor implements GeminiToolExecutor { typeof input.y !== "number" || !Number.isFinite(input.y) ) - throw new Error("missing or invalid coordinates (x, y)"); + throw new HarnessAdapterError("missing or invalid coordinates (x, y)"); return [denormalizeCoordinate(input.x, WIDTH), denormalizeCoordinate(input.y, HEIGHT)]; } diff --git a/packages/integrations/gemini-cua-sdk/src/session.ts b/packages/integrations/gemini-cua-sdk/src/session.ts index 9de4eac4b..8eb2b0efd 100644 --- a/packages/integrations/gemini-cua-sdk/src/session.ts +++ b/packages/integrations/gemini-cua-sdk/src/session.ts @@ -1,10 +1,11 @@ -import { isBrowserSessionLostError } from "@browserbasehq/stagehand-integrations/facade"; import { GoogleGenAI, Environment, type GenerateContentConfig } from "@google/genai"; import { + HarnessAdapterError, sanitizeErrorMessage, type HarnessLogger, } from "@browserbasehq/stagehand-integrations/harness"; import type { CuaFacadeTools, GeminiToolExecutor } from "./executor.js"; +import { isTerminalFacadeError, type BrowserSessionLossReader } from "./errors.js"; export type GeminiCuaUsage = { input: number; @@ -39,6 +40,7 @@ export type GeminiCuaSessionInput = { systemPrompt?: string; signal?: AbortSignal; client?: GeminiGenerateClient; + browserSessionLoss?: BrowserSessionLossReader; }; export type GeminiCuaSessionResult = { events: GeminiCuaSessionEvent[]; @@ -62,10 +64,11 @@ export const GEMINI_CUA_GENERATION_CONFIG = { } as const; export function normalizeGeminiCuaModel(model: string): string { - const bare = model.includes("/") ? model.slice(model.indexOf("/") + 1) : model; - const provider = model.includes("/") ? model.slice(0, model.indexOf("/")) : undefined; - if ((provider !== undefined && provider !== "google") || !bare.startsWith("gemini-")) - throw new Error(`Gemini Computer Use requires a gemini-* model (received "${model}")`); + // Strip the eval catalog alias only. Native model/resource names belong to + // Google, which validates support; new identifiers require no client release. + const bare = model.startsWith("google/") ? model.slice("google/".length) : model; + if (!bare.trim()) + throw new HarnessAdapterError("Gemini Computer Use requires a non-empty model identifier."); return bare; } @@ -80,7 +83,6 @@ export function createGeminiClient(options?: { apiKey?: string }): GeminiGenerat export async function runGeminiCuaSession( input: GeminiCuaSessionInput, ): Promise { - const model = normalizeGeminiCuaModel(input.model); const contents: Array> = [ { role: "user", parts: [{ text: input.prompt }] }, ]; @@ -95,6 +97,7 @@ export async function runGeminiCuaSession( let iterationError: unknown; try { + const model = normalizeGeminiCuaModel(input.model); throwIfAborted(input.signal); const client = input.client ?? createGeminiClient(); await input.facade.run("await page.setViewportSize({ width: 1288, height: 711 });"); @@ -148,20 +151,23 @@ export async function runGeminiCuaSession( .join("\n") .trim(); events.push({ type: "assistant", turn: turns, text, parts, usage: turnUsage }); - if (!candidate) throw new Error("Gemini returned no response candidate."); + if (!candidate) throw new HarnessAdapterError("Gemini returned no response candidate."); // A successful HTTP request can still be blocked or truncated. Retain // its evidence and usage, but do not execute partial calls or report it // as a completed task. if (candidate.finishReason && candidate.finishReason !== "STOP") { - throw new Error(`Gemini response ended with ${candidate.finishReason}.`); + throw new HarnessAdapterError( + `Gemini response ended with ${sanitizeErrorMessage(candidate.finishReason)}.`, + ); } contents.push({ role: "model", parts }); const calls = parts .filter(isRecord) - .map((part) => part.functionCall) - .filter(isRecord); + .filter((part) => "functionCall" in part) + .map((part) => validateFunctionCall(part.functionCall)); if (calls.length === 0) { - if (!text) throw new Error("Gemini returned neither tool calls nor an answer."); + if (!text) + throw new HarnessAdapterError("Gemini returned neither tool calls nor an answer."); finalMessage = text; status = "completed"; stopReason = candidate?.finishReason; @@ -170,8 +176,7 @@ export async function runGeminiCuaSession( const results: Record[] = []; for (const call of calls) { throwIfAborted(input.signal); - const name = typeof call.name === "string" ? call.name : ""; - const args = isRecord(call.args) ? call.args : {}; + const { name, args } = call; const id = typeof call.id === "string" ? call.id : `call_${toolCalls}`; toolCalls += 1; events.push({ type: "tool_use", turn: turns, id, name, input: args }); @@ -188,7 +193,7 @@ export async function runGeminiCuaSession( error: result.isError === true, }; events.push(resultEvent); - const observed = await observePage(input.facade, input.signal); + const observed = await observePage(input.facade, input.signal, input.browserSessionLoss); const response: Record = result.isError ? { error: result.text } : { output: result.text }; @@ -214,8 +219,8 @@ export async function runGeminiCuaSession( } } catch (error) { status = "sdk_error"; - iterationError = error; stopReason = sanitizeErrorMessage(error instanceof Error ? error.message : String(error)); + iterationError = new HarnessAdapterError(stopReason); input.logger.warn({ category: "gemini_cua", message: `session ended with error: ${stopReason}`, @@ -290,11 +295,11 @@ function readUsage(meta: Record): GeminiCuaUsage { } function throwIfAborted(signal?: AbortSignal): void { - if (signal?.aborted) throw new Error("session aborted"); + if (signal?.aborted) throw new HarnessAdapterError("session aborted"); } function isRecord(value: unknown): value is Record { - return typeof value === "object" && value !== null; + return typeof value === "object" && value !== null && !Array.isArray(value); } /** @@ -306,6 +311,7 @@ function isRecord(value: unknown): value is Record { async function observePage( facade: Pick, signal?: AbortSignal, + browserSessionLoss?: BrowserSessionLossReader, ): Promise<{ shot?: { data: string; mimeType: string }; url: string; error?: string }> { let lastError = ""; for (let attempt = 0; attempt < 2; attempt += 1) { @@ -315,11 +321,7 @@ async function observePage( const urlValue = await facade.run("return page.url();"); return { shot, url: typeof urlValue === "string" ? urlValue : "" }; } catch (error) { - if ( - signal?.aborted || - isBrowserSessionLostError(error instanceof Error ? error.message : String(error)) - ) - throw error; + if (signal?.aborted || isTerminalFacadeError(error, browserSessionLoss)) throw error; lastError = sanitizeErrorMessage(error instanceof Error ? error.message : String(error)); if (attempt === 0) await pause(1500, signal); } @@ -338,8 +340,29 @@ function pause(ms: number, signal?: AbortSignal): Promise { const aborted = () => { clearTimeout(timer); signal?.removeEventListener("abort", aborted); - reject(new Error("session aborted")); + reject(new HarnessAdapterError("session aborted")); }; signal?.addEventListener("abort", aborted, { once: true }); }); } + +/** Validate the entire response before executing any member of its call batch. */ +function validateFunctionCall(value: unknown): { + name: string; + args: Record; + id?: string; +} { + if ( + !isRecord(value) || + typeof value.name !== "string" || + !value.name.trim() || + (value.args !== undefined && !isRecord(value.args)) || + (value.id !== undefined && (typeof value.id !== "string" || !value.id.trim())) + ) + throw new HarnessAdapterError("Gemini returned a malformed function call."); + return { + name: value.name, + args: value.args ?? {}, + ...(value.id !== undefined && { id: value.id }), + }; +} diff --git a/packages/integrations/gemini-cua-sdk/src/transcript.ts b/packages/integrations/gemini-cua-sdk/src/transcript.ts index 324d51cee..27bda12bb 100644 --- a/packages/integrations/gemini-cua-sdk/src/transcript.ts +++ b/packages/integrations/gemini-cua-sdk/src/transcript.ts @@ -21,5 +21,10 @@ export function buildGeminiCuaTranscript(events: GeminiCuaSessionEvent[]): strin } function clip(value: string, max: number): string { - return value.length <= max ? value : `${value.slice(0, max - 1)}…`; + if (value.length <= max) return value; + let end = max - 1; + // Do not leave half of a UTF-16 pair at the persisted text boundary. + const last = value.charCodeAt(end - 1); + if (last >= 0xd800 && last <= 0xdbff) end -= 1; + return `${value.slice(0, end)}…`; } diff --git a/packages/integrations/gemini-cua-sdk/tests/executor.test.ts b/packages/integrations/gemini-cua-sdk/tests/executor.test.ts index 010f412a4..d896bb0ca 100644 --- a/packages/integrations/gemini-cua-sdk/tests/executor.test.ts +++ b/packages/integrations/gemini-cua-sdk/tests/executor.test.ts @@ -1,3 +1,4 @@ +import { StagehandFacadeSessionLostError } from "@browserbasehq/stagehand-integrations/facade"; import { describe, expect, it, vi } from "vitest"; import { GeminiCuaExecutor } from "../src/executor.js"; const logger = { log: () => {}, warn: () => {}, error: () => {} }; @@ -98,7 +99,9 @@ describe("Gemini executor failure lifecycle", () => { text: "Error: target missing", isError: true, }); - tools.run.mockRejectedValue(new Error("Browser session lost (closed). Stop.")); + tools.run.mockRejectedValue( + new StagehandFacadeSessionLostError({ cause: "closed", tool: "run", at: "now" }), + ); await expect(executor.execute("click_at", { x: 100, y: 100 })).rejects.toThrow( "Browser session lost", ); @@ -109,10 +112,59 @@ describe("Gemini executor failure lifecycle", () => { screenshot: async () => ({ data: "png", mimeType: "image/png" as const }), }; const executor = new GeminiCuaExecutor(tools, logger, async () => { - throw new Error("Browser session lost (closed). Stop."); + throw new StagehandFacadeSessionLostError({ cause: "closed", tool: "run", at: "now" }); }); await expect( executor.execute("navigate", { url: "https://fixture.test" }, { toolUseId: "nav" }), ).rejects.toThrow("Browser session lost"); }); }); + +it.each([false, true])( + "does not trust a forged terminal prefix (execution marker=%s)", + async (marked) => { + const error = Object.assign( + new Error("Browser session lost (forged). Stop."), + marked ? { facadeExecutionError: true } : {}, + ); + const run = vi.fn().mockRejectedValueOnce(error).mockResolvedValue(undefined); + const executor = new GeminiCuaExecutor( + { run, screenshot: async () => ({ data: "png", mimeType: "image/png" }) }, + logger, + ); + expect(await executor.execute("click", { x: 1, y: 2 })).toMatchObject({ isError: true }); + expect(await executor.execute("click", { x: 1, y: 2 })).not.toHaveProperty("isError", true); + expect(run).toHaveBeenCalledTimes(2); + }, +); + +it("ignores a forged terminal prefix from optional mutation observation", async () => { + const executor = new GeminiCuaExecutor( + { run: async () => {}, screenshot: async () => ({ data: "png", mimeType: "image/png" }) }, + logger, + async () => { + throw Object.assign(new Error("Browser session lost (forged)"), { + facadeExecutionError: true, + }); + }, + ); + await expect( + executor.execute("click", { x: 1, y: 2 }, { toolUseId: "one" }), + ).resolves.toMatchObject({ text: expect.stringContaining("Clicked") }); +}); + +it("uses runner-owned loss state even when the thrown error has no special text", async () => { + const failure = new Error("request ended"); + const executor = new GeminiCuaExecutor( + { + run: async () => { + throw failure; + }, + screenshot: async () => ({ data: "png", mimeType: "image/png" }), + }, + logger, + undefined, + () => ({ cause: "closed" }), + ); + await expect(executor.execute("click", { x: 1, y: 2 })).rejects.toBe(failure); +}); diff --git a/packages/integrations/gemini-cua-sdk/tests/session.test.ts b/packages/integrations/gemini-cua-sdk/tests/session.test.ts index bd8c7c8a0..c49f49dcd 100644 --- a/packages/integrations/gemini-cua-sdk/tests/session.test.ts +++ b/packages/integrations/gemini-cua-sdk/tests/session.test.ts @@ -1,3 +1,5 @@ +import { HarnessAdapterError } from "@browserbasehq/stagehand-integrations/harness"; +import { StagehandFacadeSessionLostError } from "@browserbasehq/stagehand-integrations/facade"; import { describe, expect, it, vi } from "vitest"; import type { CuaFacadeTools } from "../src/executor.js"; import { runGeminiCuaSession } from "../src/session.js"; @@ -42,7 +44,7 @@ describe("runGeminiCuaSession", () => { expect(result.status).toBe("sdk_error"); expect(result.stopReason).toContain(reason); expect(result.finalMessage).toBe(""); - expect(result.iterationError).toBeInstanceOf(Error); + expect(result.iterationError).toBeInstanceOf(HarnessAdapterError); expect(result.tokenUsage).toMatchObject({ input: 7, output: 3 }); expect(result.events).toHaveLength(1); expect(execute).not.toHaveBeenCalled(); @@ -261,7 +263,7 @@ describe("runGeminiCuaSession", () => { it("ends on terminal browser loss during observation without retry or later action", async () => { const execute = vi.fn(async () => ({ text: "ok" })); const screenshot = vi.fn(async () => { - throw new Error("Browser session lost (closed). Stop."); + throw new StagehandFacadeSessionLostError({ cause: "closed", tool: "screenshot", at: "now" }); }); const generateContent = vi.fn(async () => ({ candidates: [ @@ -318,3 +320,168 @@ describe("runGeminiCuaSession", () => { }, ); }); + +it.each([ + null, + [], + {}, + { name: "" }, + { name: " " }, + { name: 42 }, + { name: "wait", args: null }, + { name: "wait", args: [] }, + { name: "wait", args: "seconds=1" }, + { name: "wait", id: 3 }, +])("validates the whole function-call batch before executing (%j)", async (malformed) => { + const execute = vi.fn(async () => ({ text: "ok" })); + const result = await runGeminiCuaSession({ + prompt: "p", + model: "future-model", + logger, + maxTurns: 1, + tools: { execute }, + facade: { + run: async () => "", + screenshot: async () => ({ data: "png", mimeType: "image/png" }), + }, + client: { + generateContent: async () => ({ + candidates: [ + { + finishReason: "STOP", + content: { + parts: [{ functionCall: { name: "wait", args: {} } }, { functionCall: malformed }], + }, + }, + ], + usageMetadata: { promptTokenCount: 7 }, + }), + }, + }); + expect(result.status).toBe("sdk_error"); + expect(result.iterationError).toBeInstanceOf(HarnessAdapterError); + expect(result.stopReason).toContain("malformed function call"); + expect(result.tokenUsage.input).toBe(7); + expect(result.toolCalls).toBe(0); + expect(execute).not.toHaveBeenCalled(); +}); + +it.each([ + "future-model", + "models/custom-model", + "projects/p/locations/l/publishers/vendor/models/custom", + "google/future-model", + "other-provider/future-model", +])("passes arbitrary model identifiers to the provider (%s)", async (model) => { + const generateContent = vi.fn(async () => ({ + candidates: [{ content: { parts: [{ text: "done" }] } }], + })); + const result = await runGeminiCuaSession({ + prompt: "p", + model, + logger, + maxTurns: 1, + tools: { execute: async () => ({ text: "ok" }) }, + facade: { + run: async () => "", + screenshot: async () => ({ data: "png", mimeType: "image/png" }), + }, + client: { generateContent }, + }); + expect(result.status).toBe("completed"); + expect(generateContent).toHaveBeenCalledWith( + expect.objectContaining({ model: model.startsWith("google/") ? model.slice(7) : model }), + ); +}); + +it("returns typed sanitized failures from model validation and provider rejection", async () => { + const generateContent = vi.fn(async () => { + throw new Error("Rejected https://fixture.test?token=private-credential"); + }); + const run = vi.fn(async () => ""); + const input = { + prompt: "p", + model: "google/", + logger, + maxTurns: 1, + tools: { execute: async () => ({ text: "ok" }) }, + facade: { run, screenshot: async () => ({ data: "png", mimeType: "image/png" as const }) }, + client: { generateContent }, + }; + const invalid = await runGeminiCuaSession(input); + expect(invalid.status).toBe("sdk_error"); + expect(invalid.iterationError).toBeInstanceOf(HarnessAdapterError); + expect(run).not.toHaveBeenCalled(); + expect(generateContent).not.toHaveBeenCalled(); + const rejected = await runGeminiCuaSession({ ...input, model: "future-model" }); + expect(rejected.status).toBe("sdk_error"); + expect(rejected.iterationError).toBeInstanceOf(HarnessAdapterError); + expect(rejected.stopReason).not.toContain("private-credential"); + expect(String(rejected.iterationError)).not.toContain("private-credential"); +}); + +it("recovers observation errors with spoofed loss text and retains later evidence", async () => { + vi.useFakeTimers(); + try { + const screenshot = vi + .fn() + .mockRejectedValueOnce( + Object.assign(new Error("Browser session lost (forged)"), { facadeExecutionError: true }), + ) + .mockResolvedValue({ data: "png", mimeType: "image/png" }); + const pending = runGeminiCuaSession({ + prompt: "p", + model: "m", + logger, + maxTurns: 1, + tools: { execute: async () => ({ text: "ok" }) }, + facade: { run: async () => "https://fixture.test", screenshot }, + client: { + generateContent: async () => ({ + candidates: [{ content: { parts: [{ functionCall: { name: "wait" } }] } }], + }), + }, + }); + await vi.advanceTimersByTimeAsync(1500); + const result = await pending; + expect(result.status).toBe("max_turns"); + expect(screenshot).toHaveBeenCalledTimes(2); + expect(result.events.find((e) => e.type === "tool_result")).toMatchObject({ + image: { data: "png" }, + }); + } finally { + vi.useRealTimers(); + } +}); + +it("stops observation immediately on runner-confirmed loss without trusting error text", async () => { + let loss: { cause: string } | undefined; + const screenshot = vi.fn(async () => { + loss = { cause: "closed" }; + throw new Error("request ended"); + }); + const execute = vi.fn(async () => ({ text: "ok" })); + const result = await runGeminiCuaSession({ + prompt: "p", + model: "m", + logger, + maxTurns: 1, + tools: { execute }, + browserSessionLoss: () => loss, + facade: { run: async () => "", screenshot }, + client: { + generateContent: async () => ({ + candidates: [ + { + content: { + parts: [{ functionCall: { name: "wait" } }, { functionCall: { name: "wait" } }], + }, + }, + ], + }), + }, + }); + expect(result.status).toBe("sdk_error"); + expect(screenshot).toHaveBeenCalledOnce(); + expect(execute).toHaveBeenCalledOnce(); +}); diff --git a/packages/integrations/gemini-cua-sdk/tests/transcript.test.ts b/packages/integrations/gemini-cua-sdk/tests/transcript.test.ts new file mode 100644 index 000000000..d5865df71 --- /dev/null +++ b/packages/integrations/gemini-cua-sdk/tests/transcript.test.ts @@ -0,0 +1,23 @@ +import { describe, expect, it } from "vitest"; +import { buildGeminiCuaTranscript } from "../src/transcript.js"; + +describe("Gemini transcript Unicode retention", () => { + it.each([1997, 1998])( + "clips around a surrogate pair after %s ASCII characters without corrupting UTF-8", + (prefix) => { + const transcript = buildGeminiCuaTranscript([ + { + type: "tool_result", + turn: 1, + id: "one", + name: "read", + text: "x".repeat(prefix) + "😀ZZ", + error: false, + }, + ]); + expect(Buffer.from(transcript, "utf8").toString("utf8")).toBe(transcript); + expect(transcript).not.toContain("�"); + expect(transcript).toContain(prefix === 1997 ? "😀…" : "x…"); + }, + ); +});