diff --git a/.changeset/eval-session-ownership.md b/.changeset/eval-session-ownership.md new file mode 100644 index 0000000000..a1e2594508 --- /dev/null +++ b/.changeset/eval-session-ownership.md @@ -0,0 +1,5 @@ +--- +"@browserbasehq/stagehand": patch +--- + +Release a newly created Browserbase session when initial attachment fails, and support upload retries diff --git a/packages/evals/core/contracts/tool.ts b/packages/evals/core/contracts/tool.ts index 2971bb7662..e6dbdc0f57 100644 --- a/packages/evals/core/contracts/tool.ts +++ b/packages/evals/core/contracts/tool.ts @@ -12,6 +12,7 @@ export type ToolSurface = | "playwright_mcp" | "chrome_devtools_mcp" | "stagehand_facade" + | "stagehand_facade_legacy" | "browse_cli"; export type StartupProfile = @@ -135,6 +136,22 @@ export interface ToolStartInput { }; } +export interface BrowserSessionLoss { + cause: string; + tool?: string; + at?: string; +} + +/** MCP content returned unchanged by a runner call into its existing surface. */ +export interface RunnerToolCallResult { + content: Array< + | { type: "text"; text: string } + | { type: "image"; data: string; mimeType: string } + | Record + >; + isError?: boolean; +} + export interface ToolStartResult { session: CoreSession; /** @@ -149,6 +166,13 @@ export interface ToolStartResult { * Implementations must swallow per-field failures and must not throw. */ captureEvidence?: () => Promise; + /** Calls the same mounted surface; this must not launch another browser. */ + callTool?: ( + name: string, + args: Record, + options?: { timeoutMs?: number }, + ) => Promise; + browserSessionLoss?: () => BrowserSessionLoss | undefined; /** Releases the runtime; `captureEvidence` is invalid after this resolves. */ cleanup: () => Promise; metadata: { diff --git a/packages/evals/core/runtime/coreDeps.ts b/packages/evals/core/runtime/coreDeps.ts index d41394caf8..07b80d64a1 100644 --- a/packages/evals/core/runtime/coreDeps.ts +++ b/packages/evals/core/runtime/coreDeps.ts @@ -5,7 +5,10 @@ import { fileURLToPath } from "node:url"; type BrowserbaseConstructor = new (options: { apiKey: string }) => { extensions: { - create: (payload: { file: ReadStream }) => Promise<{ id: string }>; + create: ( + payload: { file: ReadStream }, + options?: { maxRetries?: number }, + ) => Promise<{ id: string }>; delete: ( extensionId: string, options?: { headers?: Record }, diff --git a/packages/evals/core/targets/browserbase.ts b/packages/evals/core/targets/browserbase.ts index 5c58f5cc68..7e754f8fac 100644 --- a/packages/evals/core/targets/browserbase.ts +++ b/packages/evals/core/targets/browserbase.ts @@ -2,6 +2,7 @@ import { createReadStream } from "node:fs"; import { AsyncLocalStorage } from "node:async_hooks"; import { registerActiveRunCleanup } from "../../framework/activeRunCleanup.js"; import { loadBrowserbaseSdk, resolveStagehandExtensionArchivePath } from "../runtime/coreDeps.js"; +import { evalBrowserbaseSessionOptions } from "./browserbaseSessionOptions.js"; const DEFAULT_VIEWPORT = { width: 1288, height: 711 }; const EXTENSION_SCOPE_DRAIN_TIMEOUT_MS = 5_000; @@ -22,9 +23,10 @@ const extensionScopeStorage = new AsyncLocalStorage(); async function uploadStagehandExtension( bb: BrowserbaseClient, ): Promise<{ extensionId: string; deleteUpload: () => Promise }> { - const uploaded = await bb.extensions.create({ - file: createReadStream(resolveStagehandExtensionArchivePath()), - }); + const uploaded = await bb.extensions.create( + { file: createReadStream(resolveStagehandExtensionArchivePath()) }, + { maxRetries: 0 }, + ); const extensionId = uploaded.id.trim(); if (!extensionId) { throw new Error("Browserbase extension upload returned an empty extension ID"); @@ -77,6 +79,22 @@ export async function withBrowserbaseExtensionScope(fn: () => Promise): Pr } } +/** + * One Stagehand extension upload shared by every Browserbase session the + * current run launches (the runner wraps each experiment in + * withBrowserbaseExtensionScope). Used by the facade tool so N concurrent + * facades do not each upload the archive — a burst of uploads is what + * Browserbase rejects. Returns undefined when credentials are absent. + */ +export async function acquireRunScopedStagehandExtension(): Promise< + { extensionId: string; release: () => Promise } | undefined +> { + const apiKey = process.env.BROWSERBASE_API_KEY || process.env.BB_API_KEY; + if (!apiKey) return undefined; + const Browserbase = loadBrowserbaseSdk(); + return await acquireStagehandExtension(new Browserbase({ apiKey })); +} + async function acquireStagehandExtension( bb: BrowserbaseClient, ): Promise<{ extensionId: string; release: () => Promise }> { @@ -131,14 +149,24 @@ export async function launchRunnerProvidedBrowserbaseChrome(): Promise<{ const Browserbase = loadBrowserbaseSdk(); const bb = new Browserbase({ apiKey }); + const sessionOptions = evalBrowserbaseSessionOptions(); const extension = await acquireStagehandExtension(bb); const { extensionId } = extension; const createPayload: Record = { ...(projectId ? { projectId } : {}), extensionId, + // Several CDP clients share this session (the agent's MCP server, the + // harness observer, the visible-tab probe), and each connect/disconnect + // would otherwise end it — Browserbase closes non-keepAlive sessions on + // the first disconnect (observed as "410 Gone - session not running" on + // the agent's second tool call). Release is explicit in cleanup. + keepAlive: true, + timeout: sessionOptions.timeoutSeconds, + proxies: sessionOptions.proxies, browserSettings: { viewport: DEFAULT_VIEWPORT, + verified: sessionOptions.verified, }, userMetadata: { stagehand: "true", @@ -204,7 +232,7 @@ export async function launchRunnerProvidedBrowserbaseChrome(): Promise<{ return { wsUrl: created.connectUrl, sessionId: created.id, - sessionUrl: `https://www.browserbase.com/sessions/${created.id}`, + sessionUrl: `https://www.browserbase.com/sessions/${encodeURIComponent(created.id)}`, debugUrl, extensionId, cleanup, diff --git a/packages/evals/core/targets/browserbaseSessionOptions.ts b/packages/evals/core/targets/browserbaseSessionOptions.ts new file mode 100644 index 0000000000..c2c92c4b3e --- /dev/null +++ b/packages/evals/core/targets/browserbaseSessionOptions.ts @@ -0,0 +1,55 @@ +import { EvalsError } from "../../errors.js"; + +/** + * Browserbase's project default session timeout (15 min) is shorter than many + * benchmark tasks; every eval session gets an explicit one. Seconds, mirroring + * the Browserbase API. + */ +export const DEFAULT_EVAL_BROWSERBASE_SESSION_TIMEOUT_SECONDS = 3600; +const MAX_BROWSERBASE_SESSION_TIMEOUT_SECONDS = 21_600; + +export function evalBrowserbaseSessionTimeoutSeconds(raw: string | undefined): number { + const value = raw?.trim(); + if (!value) return DEFAULT_EVAL_BROWSERBASE_SESSION_TIMEOUT_SECONDS; + const parsed = Number(value); + if ( + !/^\d+$/u.test(value) || + !Number.isSafeInteger(parsed) || + parsed < 60 || + parsed > MAX_BROWSERBASE_SESSION_TIMEOUT_SECONDS + ) { + throw new EvalsError( + `EVAL_BROWSERBASE_SESSION_TIMEOUT_SECONDS must be an integer between 60 and ${MAX_BROWSERBASE_SESSION_TIMEOUT_SECONDS} seconds (got "${value}").`, + ); + } + return parsed; +} + +export function evalBooleanEnv(raw: string | undefined, fallback: boolean): boolean { + const v = raw?.trim().toLowerCase(); + if (!v) return fallback; + if (v === "1" || v === "true" || v === "yes" || v === "on") return true; + if (v === "0" || v === "false" || v === "no" || v === "off") return false; + throw new EvalsError(`Expected a boolean env value, got "${raw}".`); +} + +/** + * Session settings shared by every Browserbase path the evals own — the + * Stagehand facade (via STAGEHAND_* env) and the runner-provided CDP target + * (playwright_mcp, chrome_devtools_mcp, playwright_code). Parity with the + * native Stagehand agent path: proxied + verified sessions, explicit timeout. + * EVAL_BROWSERBASE_PROXIES / _VERIFIED=0 to disable. + */ +export function evalBrowserbaseSessionOptions(env: NodeJS.ProcessEnv = process.env): { + timeoutSeconds: number; + proxies: boolean; + verified: boolean; +} { + return { + timeoutSeconds: evalBrowserbaseSessionTimeoutSeconds( + env.EVAL_BROWSERBASE_SESSION_TIMEOUT_SECONDS, + ), + proxies: evalBooleanEnv(env.EVAL_BROWSERBASE_PROXIES, true), + verified: evalBooleanEnv(env.EVAL_BROWSERBASE_VERIFIED, true), + }; +} diff --git a/packages/evals/core/tools/browserSessionLoss.ts b/packages/evals/core/tools/browserSessionLoss.ts new file mode 100644 index 0000000000..8ea5c804db --- /dev/null +++ b/packages/evals/core/tools/browserSessionLoss.ts @@ -0,0 +1,40 @@ +import { sanitizeErrorMessage } from "@browserbasehq/stagehand-integrations/harness"; +import type { BrowserSessionLoss } from "../contracts/tool.js"; + +/** + * Wire strings the facade stdio server emits when its browser session is gone. + * Source of truth: packages/integrations/core/src/facade/contract.ts + * (BROWSER_SESSION_LOST_ERROR_PREFIX, SESSION_LOST_TELEMETRY_PREFIX). Mirrored + * here because evals consumes the integrations package through its built dist, + * and a facade build is allowed to lag behind the runner. + */ +export const BROWSER_SESSION_LOST_ERROR_PREFIX = "Browser session lost ("; +export const SESSION_LOST_TELEMETRY_PREFIX = "stagehand_facade_session_lost "; + +export function isBrowserSessionLostError(message: string): boolean { + return message.startsWith(BROWSER_SESSION_LOST_ERROR_PREFIX); +} + +/** Extracts the cause from "Browser session lost (). ..." */ +export function browserSessionLostCause(message: string): string | undefined { + if (!isBrowserSessionLostError(message)) return undefined; + return sanitizeErrorMessage(/^Browser session lost \((.*?)\)\./su.exec(message)?.[1] ?? message); +} + +export function parseSessionLossTelemetry(line: string): BrowserSessionLoss | undefined { + if (!line.startsWith(SESSION_LOST_TELEMETRY_PREFIX)) return undefined; + try { + const parsed = JSON.parse(line.slice(SESSION_LOST_TELEMETRY_PREFIX.length)) as Record< + string, + unknown + >; + if (typeof parsed.cause !== "string") return undefined; + return { + cause: sanitizeErrorMessage(parsed.cause), + ...(typeof parsed.tool === "string" && { tool: parsed.tool }), + ...(typeof parsed.at === "string" && { at: parsed.at }), + }; + } catch { + return undefined; + } +} diff --git a/packages/evals/core/tools/registry.ts b/packages/evals/core/tools/registry.ts index 7cc36ecdd9..87b19735a0 100644 --- a/packages/evals/core/tools/registry.ts +++ b/packages/evals/core/tools/registry.ts @@ -5,12 +5,13 @@ import { ChromeDevtoolsMcpTool } from "./chrome_devtools_mcp.js"; import { PlaywrightCodeTool } from "./playwright_code.js"; import { PlaywrightMcpTool } from "./playwright_mcp.js"; import { StagehandCodeTool } from "./stagehand_code.js"; -import { StagehandFacadeTool } from "./stagehand_facade.js"; +import { StagehandFacadeTool, StagehandFacadeLegacyTool } from "./stagehand_facade.js"; import { UnderstudyCodeTool } from "./understudy_code.js"; /** Surfaces that exist only as an agent MCP mount; they have no runner-driven CoreSession (activePage() throws). */ export const AGENT_MOUNT_ONLY_TOOL_SURFACES: ReadonlySet = new Set([ "stagehand_facade", + "stagehand_facade_legacy", ]); export function isAgentMountOnlyToolSurface(toolSurface: ToolSurface): boolean { @@ -28,6 +29,7 @@ export function listCoreTools(): ToolSurface[] { // Listed here as part of the full enumeration, but agent-mount-only: // core-tier selection must use listCoreRunnableTools, which filters it. "stagehand_facade", + "stagehand_facade_legacy", "browse_cli", ]; } @@ -53,6 +55,8 @@ export function getCoreTool(toolSurface: ToolSurface): CoreTool { return new ChromeDevtoolsMcpTool(); case "stagehand_facade": return new StagehandFacadeTool(); + case "stagehand_facade_legacy": + return new StagehandFacadeLegacyTool(); case "browse_cli": return new BrowseCliTool(); default: diff --git a/packages/evals/core/tools/stagehandFacadeBridge.ts b/packages/evals/core/tools/stagehandFacadeBridge.ts index 44f269cc7d..acd8b7e7c1 100644 --- a/packages/evals/core/tools/stagehandFacadeBridge.ts +++ b/packages/evals/core/tools/stagehandFacadeBridge.ts @@ -4,6 +4,8 @@ import type { ProbeEvidence } from "stagehand-v3"; import { sanitizeErrorMessage } from "@browserbasehq/stagehand-integrations/harness"; import { EvalsError } from "../../errors.js"; import type { EvalLogger } from "../../logger.js"; +import { parseSessionLossTelemetry } from "./browserSessionLoss.js"; +import type { BrowserSessionLoss, RunnerToolCallResult } from "../contracts/tool.js"; export const STAGEHAND_FACADE_BRIDGE_PORT_ENV = "STAGEHAND_EVALS_FACADE_BRIDGE_PORT"; @@ -50,7 +52,19 @@ export interface StagehandFacadeBridge { /** Whether any agent tools/call request has passed through. */ sawAgentToolCall(): boolean; /** Raw JSON-RPC request from the runner side. */ - call(method: string, params?: Record): Promise; + call( + method: string, + params?: Record, + options?: { timeoutMs?: number }, + ): Promise; + /** Calls a tool on the existing facade, retaining content and MCP error flags. */ + callTool( + name: string, + args: Record, + options?: { timeoutMs?: number }, + ): Promise; + sessionInfo(): Promise<{ provider: "local" | "browserbase"; sessionId?: string }>; + browserSessionLoss(): BrowserSessionLoss | undefined; /** Best-effort terminal/step evidence from the shared browser. */ captureEvidence(): Promise; /** Idempotently closes the relay and facade process. */ @@ -67,6 +81,7 @@ export const FACADE_RELAY_SCRIPT = `const net=require("node:net");const port=Num type JsonRpcMessage = { id?: unknown; method?: unknown; + params?: unknown; result?: unknown; error?: unknown; }; @@ -185,6 +200,7 @@ class FacadeProcess { private readonly onLine: (line: string) => void, /** Invoked once when the process exits or fails to spawn, with the error pending callers should see. */ private readonly onExit: (error: StagehandFacadeBridgeError) => void, + private readonly onStderr: (line: string) => void, ) { this.child = spawn(server.command, server.args, { stdio: ["pipe", "pipe", "pipe"], @@ -293,9 +309,10 @@ class FacadeProcess { private rememberStderr(line: string | undefined): void { if (!line) return; - this.stderrTail.push(line); + this.onStderr(line); + this.stderrTail.push(sanitizeErrorMessage(line)); if (this.stderrTail.length > STDERR_TAIL_LINES) this.stderrTail.shift(); - this.log(line); + this.log(sanitizeErrorMessage(line)); } } @@ -327,17 +344,27 @@ class RunnerRpcClient { return typeof id === "string" && id.startsWith(RUNNER_REQUEST_ID_PREFIX); } - call(method: string, params?: Record): Promise { + call( + method: string, + params?: Record, + options?: { timeoutMs?: number }, + ): Promise { + const timeoutMs = options?.timeoutMs ?? this.timeoutMs; + if (!Number.isInteger(timeoutMs) || timeoutMs <= 0 || timeoutMs > 2_147_483_647) { + return Promise.reject( + new StagehandFacadeBridgeError("timeoutMs must be an integer between 1 and 2147483647"), + ); + } const id = `${RUNNER_REQUEST_ID_PREFIX}${this.counter++}`; return new Promise((resolve, reject) => { const timer = setTimeout(() => { this.pending.delete(id); reject( new StagehandFacadeBridgeError( - `Stagehand facade request "${method}" timed out after ${this.timeoutMs}ms`, + `Stagehand facade request "${method}" timed out after ${timeoutMs}ms`, ), ); - }, this.timeoutMs); + }, timeoutMs); this.pending.set(id, { resolve, reject, timer }); const request = { jsonrpc: "2.0", id, method, ...(params === undefined ? {} : { params }) }; if (!this.send(JSON.stringify(request))) { @@ -490,6 +517,25 @@ class AgentRelayServer { } if (message.method === "notifications/initialized") return; + if ( + message.method === "tools/call" && + message.params !== null && + typeof message.params === "object" && + "name" in message.params && + message.params.name === "session_info" + ) { + if (hasOwn(message, "id") && socket.writable) { + socket.write( + `${JSON.stringify({ + jsonrpc: "2.0", + id: message.id, + error: { code: -32601, message: "Unknown tool: session_info" }, + })}\n`, + ); + } + return; + } + let outbound = line; if (isRequest) { const bridgeId = `${AGENT_REQUEST_ID_PREFIX}${this.counter++}`; @@ -517,6 +563,8 @@ class StagehandFacadeBridgeImpl implements StagehandFacadeBridge { private initializeResult: unknown; private closed = false; private closePromise: Promise | undefined; + private runnerToolCallSeen = false; + private sessionLoss: BrowserSessionLoss | undefined; private constructor( input: StagehandFacadeBridgeInput, @@ -531,6 +579,9 @@ class StagehandFacadeBridgeImpl implements StagehandFacadeBridge { this.rpc.rejectAll(error); this.relay.disconnectAll(); }, + (line) => { + this.sessionLoss ??= parseSessionLossTelemetry(line); + }, ); this.rpc = new RunnerRpcClient((line) => this.facade.writeLine(line), requestTimeoutMs); this.relay = new AgentRelayServer( @@ -596,10 +647,14 @@ class StagehandFacadeBridgeImpl implements StagehandFacadeBridge { } sawAgentToolCall(): boolean { - return this.relay.sawToolCall; + return this.relay.sawToolCall || this.runnerToolCallSeen; } - call(method: string, params?: Record): Promise { + call( + method: string, + params?: Record, + options?: { timeoutMs?: number }, + ): Promise { if (this.closed) { return Promise.reject(new StagehandFacadeBridgeError("Stagehand facade bridge is closed")); } @@ -610,24 +665,64 @@ class StagehandFacadeBridgeImpl implements StagehandFacadeBridge { ), ); } - return this.rpc.call(method, params); + return this.rpc.call(method, params, options); + } + + async callTool( + name: string, + args: Record, + options?: { timeoutMs?: number }, + ): Promise { + this.runnerToolCallSeen = true; + const result = await this.call("tools/call", { name, arguments: args }, options); + return { + content: contentBlocks(result), + ...(result && + typeof result === "object" && + typeof (result as { isError?: unknown }).isError === "boolean" + ? { isError: (result as { isError: boolean }).isError } + : {}), + }; + } + + async sessionInfo(): Promise<{ provider: "local" | "browserbase"; sessionId?: string }> { + const result = await this.call( + "tools/call", + { name: "session_info", arguments: {} }, + { timeoutMs: 120_000 }, + ); + if (isToolError(result)) + throw new StagehandFacadeBridgeError(firstTextBlock(result) ?? "Facade session_info failed"); + const parsed = JSON.parse(firstTextBlock(result) ?? "{}") as Record; + return { + provider: parsed.provider === "browserbase" ? "browserbase" : "local", + ...(typeof parsed.sessionId === "string" && parsed.sessionId + ? { sessionId: parsed.sessionId } + : {}), + }; + } + + browserSessionLoss(): BrowserSessionLoss | undefined { + return this.sessionLoss; } async captureEvidence(): Promise { - if (!this.relay.sawToolCall) return {}; + if (!this.sawAgentToolCall()) return {}; const evidence: ProbeEvidence = {}; - const screenshot = firstImageBlock(await this.callTool("screenshot", {})); + const screenshot = firstImageBlock(await this.evidenceTool("screenshot", {})); if (screenshot) evidence.screenshot = screenshot; - const url = firstTextBlock(await this.callTool("run", { code: "return page.url();" })); + const url = firstTextBlock(await this.evidenceTool("run", { code: "return page.url();" })); if (url !== undefined && /^[a-z][a-z0-9+.-]*:/iu.test(url)) evidence.url = url; // snapshot re-hydrates the active page's element-ID map ("Every call // replaces the active page's ID map"). Capturing it mid-run would make // the agent's bracketed IDs stale, so only terminal evidence includes it. - if (this.relay.connections === 0) { - const ariaTree = firstTextBlock(await this.callTool("snapshot", { includeIframes: true })); + if (this.relay.connections === 0 && !this.runnerToolCallSeen) { + const ariaTree = firstTextBlock( + await this.evidenceTool("snapshot", { includeIframes: true }), + ); if (ariaTree !== undefined) evidence.ariaTree = ariaTree; } return evidence; @@ -649,7 +744,7 @@ class StagehandFacadeBridgeImpl implements StagehandFacadeBridge { } /** Best-effort tools/call; resolves undefined on transport error or tool error. */ - private async callTool(name: string, args: Record): Promise { + private async evidenceTool(name: string, args: Record): Promise { try { const result = await this.call("tools/call", { name, arguments: args }); return isToolError(result) ? undefined : result; @@ -666,6 +761,8 @@ class StagehandFacadeBridgeImpl implements StagehandFacadeBridge { this.log(`Dropped non-JSON facade stdout line: ${rawLine}`); return; } + // Terminal connection loss comes only from runner-owned stderr telemetry. + // Error text can be thrown by agent code and cannot establish transport state. if (!hasOwn(message, "id")) { this.relay.broadcast(rawLine); } else if (RunnerRpcClient.ownsId(message.id)) { diff --git a/packages/evals/core/tools/stagehand_code.ts b/packages/evals/core/tools/stagehand_code.ts index d4dc13093e..1044b6850c 100644 --- a/packages/evals/core/tools/stagehand_code.ts +++ b/packages/evals/core/tools/stagehand_code.ts @@ -465,7 +465,7 @@ export class StagehandCodeTool implements CoreTool { input.logger.log({ category: "stagehand_code", message: "Initialized stagehand_code Stagehand SDK runtime.", - level: 1, + level: 2, auxiliary: { startupProfile: { value: input.startupProfile, type: "string" }, environment: { value: input.environment, type: "string" }, @@ -496,6 +496,8 @@ export class StagehandCodeTool implements CoreTool { browserOwnership: "tool", connectionMode: connectionModeFromProfile(input.startupProfile), startupProfile: input.startupProfile, + ...(sdk.sessionUrl && { browserbaseSessionUrl: sdk.sessionUrl }), + ...(sdk.debugUrl && { browserbaseDebugUrl: sdk.debugUrl }), }, }; } diff --git a/packages/evals/core/tools/stagehand_facade.ts b/packages/evals/core/tools/stagehand_facade.ts index c65c4ac33d..1ff6ff4dde 100644 --- a/packages/evals/core/tools/stagehand_facade.ts +++ b/packages/evals/core/tools/stagehand_facade.ts @@ -1,5 +1,12 @@ -import { FACADE_AGENT_INSTRUCTIONS } from "@browserbasehq/stagehand-integrations/facade"; -import { buildAllowlistedEnv } from "@browserbasehq/stagehand-integrations/harness"; +import { + explicitSnapshotActionsEnabled, + facadeAgentInstructions, + LEGACY_FACADE_AGENT_INSTRUCTIONS, +} from "@browserbasehq/stagehand-integrations/facade"; +import { + buildAllowlistedEnv, + sanitizeErrorMessage, +} from "@browserbasehq/stagehand-integrations/harness"; import { fileURLToPath } from "node:url"; import { EvalsError } from "../../errors.js"; import type { Artifact, ConnectionMode } from "../contracts/results.js"; @@ -11,9 +18,21 @@ import type { StartupProfile, ToolStartInput, ToolStartResult, + ToolSurface, } from "../contracts/tool.js"; import type { TargetKind } from "../contracts/targets.js"; import { startStagehandFacadeBridge } from "./stagehandFacadeBridge.js"; +import { acquireRunScopedStagehandExtension } from "../targets/browserbase.js"; +import { + evalBooleanEnv, + evalBrowserbaseSessionTimeoutSeconds, +} from "../targets/browserbaseSessionOptions.js"; + +export { + DEFAULT_EVAL_BROWSERBASE_SESSION_TIMEOUT_SECONDS, + evalBooleanEnv, + evalBrowserbaseSessionTimeoutSeconds, +} from "../targets/browserbaseSessionOptions.js"; export class StagehandFacadeToolError extends EvalsError { override readonly name = "StagehandFacadeToolError"; @@ -84,10 +103,23 @@ class StagehandFacadeMountSession implements CoreSession { export function buildStagehandFacadeEnv( environment: ToolStartInput["environment"], + env: NodeJS.ProcessEnv = process.env, ): Record { return { - ...buildAllowlistedEnv(), + ...buildAllowlistedEnv(env), STAGEHAND_BROWSER: environment === "BROWSERBASE" ? "browserbase" : "local", + ...(explicitSnapshotActionsEnabled(env) && { EXPLICIT_SNAPSHOT_ACTIONS: "1" }), + ...(environment === "BROWSERBASE" && { + STAGEHAND_BROWSERBASE_SESSION_TIMEOUT_SECONDS: String( + evalBrowserbaseSessionTimeoutSeconds(env.EVAL_BROWSERBASE_SESSION_TIMEOUT_SECONDS), + ), + // Parity with the native Stagehand agent path (proxied + verified + // sessions); default on, EVAL_BROWSERBASE_PROXIES / _VERIFIED=0 to disable. + STAGEHAND_BROWSERBASE_PROXIES: evalBooleanEnv(env.EVAL_BROWSERBASE_PROXIES, true) ? "1" : "0", + STAGEHAND_BROWSERBASE_VERIFIED: evalBooleanEnv(env.EVAL_BROWSERBASE_VERIFIED, true) + ? "1" + : "0", + }), }; } @@ -103,12 +135,22 @@ export function buildStagehandFacadeServerSpec(environment: ToolStartInput["envi }; } +/** Same facade server, started with `--surface=legacy` so it advertises the pre-Playwright-idiom run contract. */ +export function buildStagehandFacadeLegacyServerSpec(environment: ToolStartInput["environment"]): { + command: string; + args: string[]; + env: Record; +} { + const spec = buildStagehandFacadeServerSpec(environment); + return { ...spec, args: [...spec.args, "--surface=legacy"] }; +} + function connectionModeFromProfile(startupProfile: StartupProfile): ConnectionMode { return startupProfile === "tool_create_browserbase" ? "browserbase_native" : "launch"; } export class StagehandFacadeTool implements CoreTool { - readonly id = "stagehand_facade"; + readonly id: ToolSurface = "stagehand_facade"; readonly surface = "mcp"; readonly family = "stagehand"; readonly supportedStartupProfiles: StartupProfile[] = [ @@ -125,49 +167,157 @@ export class StagehandFacadeTool implements CoreTool { args: string[]; env: Record; }; + /** + * Provides the run-shared Stagehand extension for Browserbase sessions + * (default: one upload per run). `false` makes every facade upload its own. + */ + sharedExtension?: + | false + | (() => Promise<{ extensionId: string; release: () => Promise } | undefined>); } = {}, ) {} + protected defaultServerSpec(environment: ToolStartInput["environment"]) { + return buildStagehandFacadeServerSpec(environment); + } + + protected promptInstructions(): string { + // Prompt-only variant: EXPLICIT_SNAPSHOT_ACTIONS=1 adds snapshot/screenshot + // routing guidance (the stdio server reads the same env for tool descriptions). + return facadeAgentInstructions(); + } + async start(input: ToolStartInput): Promise { const expectedProfile = input.environment === "BROWSERBASE" ? "tool_create_browserbase" : "tool_launch_local"; if (input.startupProfile !== expectedProfile) { throw new StagehandFacadeToolError( - "stagehand_facade received an invalid startup profile for the selected environment.", + `${this.id} received an invalid startup profile for the selected environment.`, ); } const session = new StagehandFacadeMountSession(); - const spec = (this.options.serverSpec ?? buildStagehandFacadeServerSpec)(input.environment); - const bridge = await startStagehandFacadeBridge({ server: spec, logger: input.logger }); + const spec = this.options.serverSpec + ? this.options.serverSpec(input.environment) + : this.defaultServerSpec(input.environment); + // Share one extension upload across the run's facades. Best effort: when + // it fails the facade falls back to uploading for itself. + let sharedExtension: { extensionId: string; release: () => Promise } | undefined; + if (input.environment === "BROWSERBASE" && this.options.sharedExtension !== false) { + try { + sharedExtension = await ( + this.options.sharedExtension ?? acquireRunScopedStagehandExtension + )(); + } catch (error) { + input.logger?.warn?.({ + category: "stagehand_facade", + level: 1, + message: `Shared Stagehand extension unavailable; the facade will upload its own: ${sanitizeErrorMessage( + error instanceof Error ? error.message : String(error), + )}`, + }); + } + } + if (sharedExtension) { + spec.env = { ...spec.env, STAGEHAND_BROWSERBASE_EXTENSION_ID: sharedExtension.extensionId }; + } + let bridge: Awaited>; + try { + bridge = await startStagehandFacadeBridge({ server: spec, logger: input.logger }); + } catch (error) { + await sharedExtension?.release().catch((): undefined => undefined); + throw error; + } if (typeof input.logger?.log === "function") { input.logger.log({ category: "stagehand_facade", - level: 1, - message: `Started runner-owned stagehand_facade bridge on 127.0.0.1:${bridge.port}.`, + level: 2, + message: `Started runner-owned ${this.id} bridge on 127.0.0.1:${bridge.port}.`, }); } + // Launch the Browserbase browser now rather than on the agent's first call + // so the session id is known before the agent starts. Local browsers have + // nothing to report and keep their lazy launch. + let browserSession: Record = {}; + try { + const info = input.environment === "BROWSERBASE" ? await bridge.sessionInfo() : undefined; + browserSession = info?.sessionId + ? { + browserbaseSessionId: info.sessionId, + browserbaseSessionUrl: `https://www.browserbase.com/sessions/${encodeURIComponent(info.sessionId)}`, + } + : {}; + } catch (error) { + if (typeof input.logger?.warn === "function") { + input.logger.warn({ + category: "stagehand_facade", + level: 1, + message: `Could not resolve the facade browser session up front: ${sanitizeErrorMessage( + error instanceof Error ? error.message : String(error), + )}`, + }); + } + } + let cleanupPromise: Promise | undefined; return { session, agentMount: { via: "mcp", - promptInstructions: FACADE_AGENT_INSTRUCTIONS, + promptInstructions: this.promptInstructions(), mcpServers: { stagehand: bridge.mcpServerSpec, }, }, captureEvidence: () => bridge.captureEvidence(), - cleanup: async () => { - await bridge.close(); - await session.close(); - }, + callTool: (name, args, options) => bridge.callTool(name, args, options), + browserSessionLoss: () => bridge.browserSessionLoss(), + cleanup: () => + (cleanupPromise ??= (async () => { + const failures: unknown[] = []; + for (const close of [ + () => bridge.close(), + () => session.close(), + () => sharedExtension?.release(), + ]) { + try { + await close(); + } catch (error) { + failures.push(error); + } + } + if (failures.length) { + throw new StagehandFacadeToolError("Failed to clean up the Stagehand facade.", { + cause: new AggregateError(failures, "Facade cleanup failures"), + }); + } + })()), metadata: { environment: input.environment === "BROWSERBASE" ? "browserbase" : "local", browserOwnership: "tool", connectionMode: connectionModeFromProfile(input.startupProfile), startupProfile: input.startupProfile, + promptVariant: explicitSnapshotActionsEnabled() ? "explicit_snapshot_actions" : "default", facadeBridgePort: bridge.port, + ...browserSession, }, }; } } + +/** + * The facade surface as it shipped before the Playwright-idiom prompt: same + * server, same snapshot/screenshot tools, but the earlier run description and + * agent instructions. Kept under its own id so trajectories from the two + * prompts are never compared as one surface. + */ +export class StagehandFacadeLegacyTool extends StagehandFacadeTool { + override readonly id: ToolSurface = "stagehand_facade_legacy"; + + protected override defaultServerSpec(environment: ToolStartInput["environment"]) { + return buildStagehandFacadeLegacyServerSpec(environment); + } + + protected override promptInstructions(): string { + return LEGACY_FACADE_AGENT_INSTRUCTIONS; + } +} diff --git a/packages/evals/tests/cli.test.ts b/packages/evals/tests/cli.test.ts index 7aca4500cd..fb106d7bb3 100644 --- a/packages/evals/tests/cli.test.ts +++ b/packages/evals/tests/cli.test.ts @@ -1,5 +1,5 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ -import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import { describe, it, expect, beforeAll, afterAll, onTestFinished } from "vitest"; import { execFile } from "node:child_process"; import { promisify } from "node:util"; import fs from "node:fs"; @@ -9,6 +9,7 @@ const exec = promisify(execFile); const repoRoot = path.resolve(__dirname, "..", "..", ".."); const CLI_PATH = path.join(repoRoot, "packages", "evals", "cli.ts"); const SOURCE_CONFIG = path.join(repoRoot, "packages", "evals", "evals.config.json"); +const CLI_CHILD_TIMEOUT_MS = 15_000; // File-level snapshot/restore: any `evals run …` invocation through the // real CLI writes `_meta.firstRunCompletedAt` into the source config @@ -24,15 +25,20 @@ afterAll(() => { async function runCli(args: string[]): Promise<{ stdout: string; stderr: string; code: number }> { try { - const { stdout, stderr } = await exec( - process.execPath, - ["--import", "tsx", CLI_PATH, ...args], - { - cwd: repoRoot, - timeout: 15_000, - env: { ...process.env, NODE_NO_WARNINGS: "1" }, - }, - ); + const execution = exec(process.execPath, ["--import", "tsx", CLI_PATH, ...args], { + cwd: repoRoot, + timeout: CLI_CHILD_TIMEOUT_MS, + killSignal: "SIGKILL", + env: { ...process.env, NODE_NO_WARNINGS: "1" }, + }); + // A test timeout must also stop its own CLI child. SIGTERM enters the + // CLI's async cleanup path, which is not a bounded subprocess deadline. + onTestFinished(() => { + if (execution.child.exitCode === null && execution.child.signalCode === null) { + execution.child.kill("SIGKILL"); + } + }); + const { stdout, stderr } = await execution; return { stdout, stderr, code: 0 }; } catch (err: any) { return { @@ -55,15 +61,19 @@ function readSourceWelcomeCompletedAt(): string | undefined { } describe("CLI entrypoint", () => { - it("shows help", async () => { - const { stdout, code } = await runCli(["-h"]); - expect(code).toBe(0); - expect(stdout).toContain("Commands:"); - expect(stdout).toContain("run"); - expect(stdout).toContain("list"); - expect(stdout).toContain("config"); - expect(stdout).toContain("experiments"); - }); + it( + "shows help", + async () => { + const { stdout, code } = await runCli(["-h"]); + expect(code).toBe(0); + expect(stdout).toContain("Commands:"); + expect(stdout).toContain("run"); + expect(stdout).toContain("list"); + expect(stdout).toContain("config"); + expect(stdout).toContain("experiments"); + }, + CLI_CHILD_TIMEOUT_MS + 2_000, + ); it("shows experiments overview help", async () => { const { stdout, code } = await runCli(["experiments"]); diff --git a/packages/evals/tests/core/browserbase-session-options.test.ts b/packages/evals/tests/core/browserbase-session-options.test.ts new file mode 100644 index 0000000000..d67d124391 --- /dev/null +++ b/packages/evals/tests/core/browserbase-session-options.test.ts @@ -0,0 +1,59 @@ +import { describe, expect, it } from "vitest"; +import { stagehandFacadeConfigFromEnv } from "@browserbasehq/stagehand-integrations/facade"; +import { + evalBooleanEnv, + evalBrowserbaseSessionOptions, + evalBrowserbaseSessionTimeoutSeconds, +} from "../../core/targets/browserbaseSessionOptions.js"; +import { browserSessionLostCause } from "../../core/tools/browserSessionLoss.js"; + +// Browserbase API contract: https://docs.browserbase.com/reference/api/create-a-session +// Package parsers stay independent; this test prevents their public ranges drifting. +describe("Browserbase session settings", () => { + it.each([undefined, "60", "3600", "21600"])( + "agrees with facade timeout validation for %s", + (raw) => { + const facade = stagehandFacadeConfigFromEnv({ + BROWSERBASE_API_KEY: "test", + STAGEHAND_BROWSERBASE_SESSION_TIMEOUT_SECONDS: raw, + }); + expect(facade.browser.launchOptions).toMatchObject({ + timeout: evalBrowserbaseSessionTimeoutSeconds(raw), + }); + }, + ); + it.each(["0", "59", "21601", "NaN", "60.5", "-1", "9007199254740992"])( + "rejects invalid timeout %s in both packages", + (raw) => { + expect(() => evalBrowserbaseSessionTimeoutSeconds(raw)).toThrow(); + expect(() => + stagehandFacadeConfigFromEnv({ + BROWSERBASE_API_KEY: "test", + STAGEHAND_BROWSERBASE_SESSION_TIMEOUT_SECONDS: raw, + }), + ).toThrow(); + }, + ); + it.each(["1", "true", "yes", "on", " TRUE "])("accepts enabled boolean %s", (raw) => + expect(evalBooleanEnv(raw, false)).toBe(true), + ); + it.each(["0", "false", "no", "off", " OFF "])("accepts disabled boolean %s", (raw) => + expect(evalBooleanEnv(raw, true)).toBe(false), + ); + it("uses defaults and rejects unknown boolean values", () => { + expect(evalBrowserbaseSessionOptions({})).toEqual({ + timeoutSeconds: 3600, + proxies: true, + verified: true, + }); + expect(evalBooleanEnv(" ", false)).toBe(false); + expect(() => evalBooleanEnv("maybe", true)).toThrow(); + }); + it("extracts multiline loss causes without the surrounding instruction", () => { + expect( + browserSessionLostCause( + "Browser session lost (first\nsecond https://example.test?apiKey=secret123). Stop.", + ), + ).toBe("first\nsecond https://example.test?apiKey=[redacted]"); + }); +}); diff --git a/packages/evals/tests/core/browserbase-target.test.ts b/packages/evals/tests/core/browserbase-target.test.ts index 0a8c9a1193..fc058528c8 100644 --- a/packages/evals/tests/core/browserbase-target.test.ts +++ b/packages/evals/tests/core/browserbase-target.test.ts @@ -46,6 +46,15 @@ describe("runner-provided Browserbase target", () => { process.env = { ...originalEnv }; }); + it("validates configuration before acquiring an extension", async () => { + process.env.EVAL_BROWSERBASE_SESSION_TIMEOUT_SECONDS = "59"; + const { launchRunnerProvidedBrowserbaseChrome } = + await import("../../core/targets/browserbase.js"); + await expect(launchRunnerProvidedBrowserbaseChrome()).rejects.toThrow("between 60 and 21600"); + expect(extensionCreateMock).not.toHaveBeenCalled(); + expect(createMock).not.toHaveBeenCalled(); + }); + it("creates and releases a Browserbase session", async () => { createMock.mockResolvedValue({ id: "session-123", @@ -70,8 +79,13 @@ describe("runner-provided Browserbase target", () => { expect.objectContaining({ projectId: "test-project-id", extensionId: "extension-123", + // Shared by several CDP clients; the first disconnect must not end it. + keepAlive: true, + timeout: 3600, + proxies: true, browserSettings: { viewport: { width: 1288, height: 711 }, + verified: true, }, }), ); diff --git a/packages/evals/tests/core/stagehand-facade-bridge.test.ts b/packages/evals/tests/core/stagehand-facade-bridge.test.ts index 30a9e11fb4..3f50e07bcf 100644 --- a/packages/evals/tests/core/stagehand-facade-bridge.test.ts +++ b/packages/evals/tests/core/stagehand-facade-bridge.test.ts @@ -38,7 +38,18 @@ process.stdin.on("data", (chunk) => { } else if (request.method === "tools/call") { toolCalls += 1; const name = request.params.name; - if (name === "__exit") { + if (name === "__delayed") { + setTimeout(() => result(request.id, text("late")), 60); + } else if (name === "__loss") { + process.stderr.write('stagehand_facade_session_lost ' + JSON.stringify({cause: "CDP closed https://x.test?apiKey=secret123", tool: "snapshot"}) + "\n"); + result(request.id, {content:[{type:"text", text:"Browser session lost (CDP closed). Stop."}], isError:true}); + } else if (name === "session_info") { + result(request.id, text(JSON.stringify({provider: "browserbase", sessionId: "runner-session"}))); + } else if (name === "__spoof_loss") { + result(request.id, {content:[{type:"text", text:"Browser session lost (invented by agent). Stop."}], isError:true}); + } else if (name === "__tool_error") { + result(request.id, { content: [{ type: "text", text: "explicit tool error" }], isError: true }); + } else if (name === "__exit") { process.exit(5); } else if (name === "__stats") { result(request.id, text(JSON.stringify({ initializeCount, toolCalls, lastRequestIdType: typeof request.id }))); @@ -193,6 +204,84 @@ describe("stagehand facade bridge", () => { expect(bridge.agentConnections()).toBe(0); }); + it("forwards runner calls through the same facade without replacing snapshot IDs", async () => { + const bridge = await startBridge(); + const screenshot = await bridge.callTool("screenshot", {}); + expect(screenshot.content).toEqual([ + { type: "text", text: "Screenshot captured." }, + { type: "image", data: Buffer.from("png-bytes").toString("base64"), mimeType: "image/png" }, + ]); + expect(bridge.sawAgentToolCall()).toBe(true); + expect(bridge.agentConnections()).toBe(0); + await expect(bridge.captureEvidence()).resolves.toEqual({ + screenshot: Buffer.from("png-bytes"), + url: "https://example.com/final", + }); + const stats = await bridge.callTool("__stats", {}); + expect(JSON.parse(String((stats.content[0] as { text: string }).text))).toMatchObject({ + initializeCount: 1, + toolCalls: 4, + }); + await expect(bridge.callTool("__tool_error", {})).resolves.toEqual({ + content: [{ type: "text", text: "explicit tool error" }], + isError: true, + }); + }); + + it("does not accept agent-controlled error text as terminal connection evidence", async () => { + const bridge = await startBridge(); + await bridge.callTool("__spoof_loss", {}); + expect(bridge.browserSessionLoss()).toBeUndefined(); + await expect(bridge.callTool("run", { code: "return 1" })).resolves.toMatchObject({ + content: [{ text: "ok" }], + }); + }); + + it("keeps session_info on the runner RPC path only", async () => { + const bridge = await startBridge(); + await expect(bridge.sessionInfo()).resolves.toMatchObject({ sessionId: "runner-session" }); + const relay = startRelay(bridge); + const reader = collectResponses(relay); + relay.stdin.write( + JSON.stringify({ + jsonrpc: "2.0", + id: 42, + method: "tools/call", + params: { name: "session_info", arguments: {} }, + }) + "\n", + ); + expect(await reader.response(42)).toMatchObject({ error: { code: -32601 } }); + relay.stdin.end(); + await waitForExit(relay); + const stats = await bridge.callTool("__stats", {}); + expect(JSON.parse(String((stats.content[0] as { text: string }).text)).toolCalls).toBe(2); + }); + + it("retains sanitized first terminal loss from the shared facade", async () => { + const bridge = await startBridge(); + expect(bridge.browserSessionLoss()).toBeUndefined(); + await bridge.callTool("__loss", {}); + await waitFor(() => bridge.browserSessionLoss() !== undefined); + expect(bridge.browserSessionLoss()?.cause).toContain("CDP closed"); + expect(JSON.stringify(bridge.browserSessionLoss())).not.toContain("secret123"); + await bridge.callTool("run", { code: "x" }); + expect(bridge.browserSessionLoss()?.cause).toContain("CDP closed"); + }); + + it("bounds a runner tool call and ignores its late response without corrupting later calls", async () => { + const bridge = await startBridge(); + await expect(bridge.callTool("__delayed", {}, { timeoutMs: 10 })).rejects.toThrow("after 10ms"); + await expect(bridge.callTool("__delayed", {}, { timeoutMs: 200 })).resolves.toEqual({ + content: [{ type: "text", text: "late" }], + }); + await expect(bridge.callTool("run", { code: "x" })).resolves.toEqual({ + content: [{ type: "text", text: "ok" }], + }); + await expect(bridge.callTool("run", {}, { timeoutMs: 2_147_483_648 })).rejects.toThrow( + "timeoutMs must be", + ); + }); + it("relays agent MCP traffic and captures step then terminal evidence", async () => { const bridge = await startBridge(); const relay = startRelay(bridge); diff --git a/packages/evals/tests/core/stagehand-facade-ownership.test.ts b/packages/evals/tests/core/stagehand-facade-ownership.test.ts new file mode 100644 index 0000000000..8b1200b7a2 --- /dev/null +++ b/packages/evals/tests/core/stagehand-facade-ownership.test.ts @@ -0,0 +1,102 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import type { EvalLogger } from "../../logger.js"; + +const { startBridge, bridge, release } = vi.hoisted(() => ({ + startBridge: vi.fn(), + release: vi.fn(async () => undefined), + bridge: { + port: 1234, + mcpServerSpec: { command: "node", args: [] as string[], env: {} }, + close: vi.fn(async () => undefined), + sessionInfo: vi.fn(async () => ({ provider: "browserbase", sessionId: "owned-session" })), + captureEvidence: vi.fn(async () => ({})), + callTool: vi.fn(), + browserSessionLoss: vi.fn(), + }, +})); +vi.mock("../../core/tools/stagehandFacadeBridge.js", () => ({ + startStagehandFacadeBridge: startBridge, +})); +import { StagehandFacadeTool } from "../../core/tools/stagehand_facade.js"; + +const input = { + environment: "BROWSERBASE" as const, + startupProfile: "tool_create_browserbase" as const, + logger: {} as EvalLogger, +}; +const makeTool = () => + new StagehandFacadeTool({ + serverSpec: () => ({ command: "node", args: [] as string[], env: {} }), + sharedExtension: async () => ({ extensionId: "owned-extension", release }), + }); + +beforeEach(() => { + vi.clearAllMocks(); + startBridge.mockResolvedValue(bridge); + bridge.close.mockResolvedValue(undefined); +}); + +describe("facade ownership cleanup", () => { + it("releases the upload lease when bridge startup fails", async () => { + const failure = new Error("startup failed"); + startBridge.mockRejectedValueOnce(failure); + await expect(makeTool().start(input)).rejects.toBe(failure); + expect(release).toHaveBeenCalledOnce(); + }); + + it("releases the upload even when bridge cleanup fails and settles cleanup only once", async () => { + const running = await makeTool().start(input); + const failure = new Error("bridge cleanup failed"); + bridge.close.mockRejectedValueOnce(failure); + const sessionClose = vi.spyOn(running.session, "close"); + const first = running.cleanup(); + expect(running.cleanup()).toBe(first); + await expect(first).rejects.toMatchObject({ + message: "Failed to clean up the Stagehand facade.", + cause: { errors: [failure] }, + }); + expect(sessionClose).toHaveBeenCalledOnce(); + expect(release).toHaveBeenCalledOnce(); + }); + + it("releases the upload when session cleanup fails", async () => { + const running = await makeTool().start(input); + const failure = new Error("session cleanup failed"); + vi.spyOn(running.session, "close").mockRejectedValue(failure); + await expect(running.cleanup()).rejects.toMatchObject({ + message: "Failed to clean up the Stagehand facade.", + cause: { errors: [failure] }, + }); + expect(release).toHaveBeenCalledOnce(); + }); + + it("retains every cleanup failure behind a fixed public diagnostic", async () => { + const running = await makeTool().start(input); + const first = new Error("apiKey=private-key"); + const second = new Error("session failed"); + bridge.close.mockRejectedValueOnce(first); + vi.spyOn(running.session, "close").mockRejectedValueOnce(second); + await expect(running.cleanup()).rejects.toMatchObject({ + message: "Failed to clean up the Stagehand facade.", + cause: { errors: [first, second] }, + }); + expect(release).toHaveBeenCalledOnce(); + }); + + it("passes the shared upload to the existing facade and records its session identity", async () => { + const running = await makeTool().start(input); + expect(startBridge).toHaveBeenCalledWith( + expect.objectContaining({ + server: expect.objectContaining({ + env: { STAGEHAND_BROWSERBASE_EXTENSION_ID: "owned-extension" }, + }), + }), + ); + expect(running.metadata.browserbaseSessionId).toBe("owned-session"); + expect(running.metadata.browserbaseSessionUrl).toBe( + "https://www.browserbase.com/sessions/owned-session", + ); + await running.cleanup(); + expect(release).toHaveBeenCalledOnce(); + }); +}); diff --git a/packages/evals/tests/core/stagehand-facade.test.ts b/packages/evals/tests/core/stagehand-facade.test.ts index c484a51085..6923ad1b60 100644 --- a/packages/evals/tests/core/stagehand-facade.test.ts +++ b/packages/evals/tests/core/stagehand-facade.test.ts @@ -52,6 +52,14 @@ afterEach(() => { }); describe("stagehand facade tool surface", () => { + it("uses only the supplied environment for the child allowlist", () => { + process.env.BROWSERBASE_API_KEY = "ambient-secret"; + expect(buildStagehandFacadeEnv("LOCAL", { STAGEHAND_MODEL_NAME: "test-model" })).toEqual({ + STAGEHAND_BROWSER: "local", + STAGEHAND_MODEL_NAME: "test-model", + }); + }); + it("is registered as agent-mount-only", () => { expect(listCoreTools()).toContain("stagehand_facade"); // Never selectable for core-tier runs: its CoreSession throws on every @@ -178,6 +186,9 @@ describe("stagehand facade tool surface", () => { STAGEHAND_BROWSER: "browserbase", STAGEHAND_MODEL_API_KEY: "model-secret", BROWSERBASE_PROJECT_ID: "project-id", + STAGEHAND_BROWSERBASE_SESSION_TIMEOUT_SECONDS: "3600", + STAGEHAND_BROWSERBASE_PROXIES: "1", + STAGEHAND_BROWSERBASE_VERIFIED: "1", }); }); diff --git a/packages/evals/tests/tui/helpImports.test.ts b/packages/evals/tests/tui/helpImports.test.ts new file mode 100644 index 0000000000..5b04759434 --- /dev/null +++ b/packages/evals/tests/tui/helpImports.test.ts @@ -0,0 +1,35 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { buildCommandTree, dispatch } from "../../tui/commandTree.js"; + +vi.mock("../../framework/benchHarness.js", () => { + throw new Error("Help must not initialize the harness runtime"); +}); +vi.mock("../../core/tools/registry.js", () => { + throw new Error("Help must not initialize the tool runtime"); +}); + +afterEach(() => vi.restoreAllMocks()); + +describe("help without runtime imports", () => { + it.each([ + { args: ["--help"], expected: "Commands:" }, + { args: ["list", "--help"], expected: "evals list" }, + { args: ["new", "--help"], expected: "evals new" }, + { args: ["experiments", "--help"], expected: "evals experiments" }, + { args: ["config", "tracing", "--help"], expected: "evals config tracing" }, + ])("prints $args with harness and tool modules unavailable", async ({ args, expected }) => { + const log = vi.spyOn(console, "log").mockImplementation(() => {}); + const getRegistry = vi.fn(async () => { + throw new Error("Help must not discover tasks"); + }); + await dispatch(buildCommandTree(), args, { + entryDir: "/unused", + getRegistry, + setRegistry: vi.fn(), + abortRef: null, + contextPath: null, + }); + expect(log.mock.calls.flat().join("\n")).toContain(expected); + expect(getRegistry).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/evals/tui/commands/config.ts b/packages/evals/tui/commands/config.ts index 8a2420f7a9..9fd9eee0dc 100644 --- a/packages/evals/tui/commands/config.ts +++ b/packages/evals/tui/commands/config.ts @@ -177,7 +177,7 @@ export async function handleConfig(args: string[], entryDir: string): Promise { + const { listBenchHarnesses, listBenchHarnessesForTaskKind } = + await import("../../framework/benchHarness.js"); const suiteHarness = listBenchHarnessesForTaskKind("suite")[0]; print([ "", @@ -153,7 +153,8 @@ export function printNewHelp(): void { ]); } -export function printConfigHelp(): void { +export async function printConfigHelp(): Promise { + const { listCoreRunnableTools } = await import("../../core/tools/registry.js"); print([ "", ` ${dustyCyanHeader("evals config")} ${dim("[subcommand]")}`, @@ -194,7 +195,8 @@ export function printConfigHelp(): void { ]); } -export function printConfigCoreHelp(): void { +export async function printConfigCoreHelp(): Promise { + const { listCoreRunnableTools } = await import("../../core/tools/registry.js"); print([ "", ` ${dustyCyanHeader("evals config core")} ${dim("[subcommand]")}`, diff --git a/packages/integrations/core/src/facade/config.ts b/packages/integrations/core/src/facade/config.ts index 14c5ab9a57..a3d28a424f 100644 --- a/packages/integrations/core/src/facade/config.ts +++ b/packages/integrations/core/src/facade/config.ts @@ -9,6 +9,11 @@ export class StagehandFacadeConfigError extends Error { override readonly name = "StagehandFacadeConfigError"; } +/** Browserbase's project default is 15 minutes, shorter than most agent tasks. */ +export const DEFAULT_BROWSERBASE_SESSION_TIMEOUT_SECONDS = 3600; +/** Browserbase rejects longer sessions (6 hours). */ +export const MAX_BROWSERBASE_SESSION_TIMEOUT_SECONDS = 21_600; + export type StagehandFacadeConfig = { browser: | { type: "local"; launchOptions: LocalBrowserLaunchOptions } @@ -34,6 +39,22 @@ export function stagehandFacadeConfigFromEnv( 'BROWSERBASE_API_KEY is required when STAGEHAND_BROWSER="browserbase".', ); } + const browserbaseOptions = + browserType === "browserbase" + ? { + timeout: browserbaseSessionTimeoutSeconds( + env.STAGEHAND_BROWSERBASE_SESSION_TIMEOUT_SECONDS, + ), + proxies: booleanEnv(env.STAGEHAND_BROWSERBASE_PROXIES, "STAGEHAND_BROWSERBASE_PROXIES"), + verified: booleanEnv( + env.STAGEHAND_BROWSERBASE_VERIFIED, + "STAGEHAND_BROWSERBASE_VERIFIED", + ), + } + : undefined; + // A host that launches many facades at once uploads the Stagehand extension + // once and hands every facade the id; otherwise each session uploads its own. + const extensionId = nonEmpty(env.STAGEHAND_BROWSERBASE_EXTENSION_ID); const explicitModelName = nonEmpty(env.STAGEHAND_MODEL_NAME); const explicitModelApiKey = nonEmpty(env.STAGEHAND_MODEL_API_KEY); @@ -77,6 +98,20 @@ export function stagehandFacadeConfigFromEnv( launchOptions: { apiKey: browserbaseApiKey!, ...(browserbaseProjectId ? { projectId: browserbaseProjectId } : {}), + timeout: browserbaseOptions!.timeout, + // The facade owns shutdown explicitly, including sessions whose + // client transport has already closed. + keepAlive: true, + // Bot-wall parity with Stagehand's native agent path, which runs + // proxied + verified sessions; unproxied facade sessions were + // blocked (Akamai/PerimeterX) on sites the native path reached. + ...(browserbaseOptions!.proxies !== undefined + ? { proxies: browserbaseOptions!.proxies } + : {}), + ...(browserbaseOptions!.verified !== undefined + ? { browserSettings: { verified: browserbaseOptions!.verified } } + : {}), + ...(extensionId !== undefined ? { extensionId } : {}), }, } : { type: "local", launchOptions: { headless: false } }, @@ -84,6 +119,23 @@ export function stagehandFacadeConfigFromEnv( }; } +function browserbaseSessionTimeoutSeconds(raw: string | undefined): number { + const value = nonEmpty(raw); + if (value === undefined) return DEFAULT_BROWSERBASE_SESSION_TIMEOUT_SECONDS; + const parsed = Number(value); + if ( + !/^\d+$/u.test(value) || + !Number.isSafeInteger(parsed) || + parsed < 60 || + parsed > MAX_BROWSERBASE_SESSION_TIMEOUT_SECONDS + ) { + throw new StagehandFacadeConfigError( + `STAGEHAND_BROWSERBASE_SESSION_TIMEOUT_SECONDS must be an integer between 60 and ${MAX_BROWSERBASE_SESSION_TIMEOUT_SECONDS} seconds (got "${value}").`, + ); + } + return parsed; +} + function nonEmpty(value: string | undefined): string | undefined { const trimmed = value?.trim(); return trimmed ? trimmed : undefined; @@ -113,3 +165,11 @@ function providerApiKey(provider: string | undefined, env: NodeJS.ProcessEnv): s return undefined; } } + +function booleanEnv(value: string | undefined, name: string): boolean | undefined { + const trimmed = value?.trim().toLowerCase(); + if (!trimmed) return undefined; + if (trimmed === "1" || trimmed === "true" || trimmed === "yes" || trimmed === "on") return true; + if (trimmed === "0" || trimmed === "false" || trimmed === "no" || trimmed === "off") return false; + throw new StagehandFacadeConfigError(`${name} must be a boolean (got "${value}").`); +} diff --git a/packages/integrations/core/src/facade/contract.ts b/packages/integrations/core/src/facade/contract.ts index 97abd1a13f..fcf27ed221 100644 --- a/packages/integrations/core/src/facade/contract.ts +++ b/packages/integrations/core/src/facade/contract.ts @@ -39,11 +39,25 @@ const actionSchema = (op: string, extra: Record> }); export const RUN_TOOL_DESCRIPTION = + 'Browse and automate websites in the persistent browser by executing JavaScript against a Playwright-shaped API. The code runs inside an async function with page, context, and browser in scope (Playwright Page, BrowserContext, and Browser); use await directly and return a JSON-serializable value when useful. Navigate with await page.goto("https://example.com"); there is no separate navigate or start tool. Alternatively, provide a batch of actions using IDs from the latest snapshot. Provide exactly one of code or actions. Each action must use "op" (never "kind") and "id" (never "ref"). Copy the bracketed snapshot ID as a string. Examples: {"actions":[{"op":"click","id":"1-42"}]}, {"actions":[{"op":"fill","id":"2-14","value":"Miami"}]}, {"actions":[{"op":"select","id":"3-9","values":"Lowest price"}]}.'; + +/** + * Run-tool description of the pre-Playwright-idiom facade surface, kept + * byte-identical so `--surface=legacy` hosts reproduce the earlier contract. + */ +export const LEGACY_RUN_TOOL_DESCRIPTION = 'Browse and automate websites in the persistent Stagehand browser. Navigate with JavaScript such as await page.goto("https://example.com"); there is no separate navigate or start tool. Execute either a JavaScript workflow against the Stagehand Playwright facade or a batch of actions using IDs from the latest snapshot. Provide exactly one of code or actions. Each action must use "op" (never "kind") and "id" (never "ref"). Copy the bracketed snapshot ID as a string. Examples: {"actions":[{"op":"click","id":"1-42"}]}, {"actions":[{"op":"fill","id":"2-14","value":"Miami"}]}, {"actions":[{"op":"select","id":"3-9","values":"Lowest price"}]}.'; export const SNAPSHOT_TOOL_DESCRIPTION = "Capture the active page's Stagehand accessibility tree and hydrate its displayed IDs for subsequent run actions. Every call replaces the active page's ID map."; +/** + * Runner-side tool, deliberately absent from tools/list. Launches the browser + * if needed and reports `{ provider, sessionId? }` so the harness can log the + * Browserbase session before the agent's first call. + */ +export const SESSION_INFO_TOOL_NAME = "session_info"; + export const SCREENSHOT_TOOL_DESCRIPTION = 'Capture a screenshot of the active page. For size-constrained MCP clients, prefer a viewport JPEG: {"type":"jpeg","quality":40,"fullPage":false}.'; @@ -117,12 +131,88 @@ export const FACADE_TOOLS = [ }, ] as const; +export const FACADE_LEGACY_TOOLS = [ + { name: "run", description: LEGACY_RUN_TOOL_DESCRIPTION, inputSchema: RUN_INPUT_SCHEMA }, + FACADE_TOOLS[1], + FACADE_TOOLS[2], +] as const; + +export type FacadeSurface = "playwright" | "legacy"; + +const SURFACE_FLAG = "--surface="; + +/** `--surface=legacy` selects the pre-Playwright-idiom run description; default is "playwright". */ +export function facadeSurfaceFromArgs(args: string[]): FacadeSurface { + const value = args.find((arg) => arg.startsWith(SURFACE_FLAG)); + if (value === undefined) return "playwright"; + const surface = value.slice(SURFACE_FLAG.length); + if (surface !== "playwright" && surface !== "legacy") { + throw new Error(`${SURFACE_FLAG} must be "playwright" or "legacy".`); + } + return surface; +} + +export function facadeToolsForSurface(surface: FacadeSurface) { + return surface === "legacy" ? FACADE_LEGACY_TOOLS : FACADE_TOOLS; +} + +/** + * Prompt-only variant (env EXPLICIT_SNAPSHOT_ACTIONS=1): same tools and + * runtime, but the instructions and tool descriptions route simple + * interactions through snapshot IDs and ask for a screenshot before + * reporting. Without it the Playwright-idiom text leads agents to code-mode + * for ~99% of run calls (measured 2026-08-31: 16 ref-action calls in 1,199). + */ +export const SNAPSHOT_ACTIONS_WORKFLOW = `Workflow: after navigating or whenever the page changes, call snapshot first and read the tree. For simple interactions on visible elements — click, fill, select, press — act by snapshot ID through run "actions" (one round trip, no selector guessing). Use run "code" for multi-step logic, data extraction, waits, or anything the snapshot does not expose. Snapshot IDs are valid only for the latest snapshot of the active page; snapshot again after navigation. Call screenshot before declaring success and whenever layout or images matter to the task.`; + +export const SNAPSHOT_TOOL_DESCRIPTION_EXPLICIT = `${SNAPSHOT_TOOL_DESCRIPTION} Call it after every navigation or page change and before acting by ID.`; +export const SCREENSHOT_TOOL_DESCRIPTION_EXPLICIT = `${SCREENSHOT_TOOL_DESCRIPTION} Use it to verify the visible state before reporting done, on visual or canvas content, and when text evidence is ambiguous.`; +export const RUN_TOOL_DESCRIPTION_EXPLICIT = `${RUN_TOOL_DESCRIPTION} Prefer "actions" with snapshot IDs for simple clicks/fills/selects on visible elements; use "code" for multi-step logic and extraction.`; + +export function explicitSnapshotActionsEnabled(env: NodeJS.ProcessEnv = process.env): boolean { + const v = env.EXPLICIT_SNAPSHOT_ACTIONS?.trim().toLowerCase(); + return v === "1" || v === "true" || v === "yes" || v === "on"; +} + +export const FACADE_TOOLS_EXPLICIT_SNAPSHOT = FACADE_TOOLS.map((tool) => ({ + ...tool, + description: + tool.name === "run" + ? RUN_TOOL_DESCRIPTION_EXPLICIT + : tool.name === "snapshot" + ? SNAPSHOT_TOOL_DESCRIPTION_EXPLICIT + : SCREENSHOT_TOOL_DESCRIPTION_EXPLICIT, +})) as unknown as typeof FACADE_TOOLS; + +export function facadeToolsFor(surface: FacadeSurface, env: NodeJS.ProcessEnv = process.env) { + if (surface === "legacy") return FACADE_LEGACY_TOOLS; + return explicitSnapshotActionsEnabled(env) ? FACADE_TOOLS_EXPLICIT_SNAPSHOT : FACADE_TOOLS; +} + +export function facadeAgentInstructions(env: NodeJS.ProcessEnv = process.env): string { + return explicitSnapshotActionsEnabled(env) + ? `${FACADE_AGENT_INSTRUCTIONS}\n\n${SNAPSHOT_ACTIONS_WORKFLOW}` + : FACADE_AGENT_INSTRUCTIONS; +} + /** * Canonical agent system prompt for the facade tool surface. Host examples * (Eve, Vercel AI SDK, deepagents) should use this text rather than authoring * their own so agent guidance stays identical across frameworks. */ -export const FACADE_AGENT_INSTRUCTIONS = `You control one persistent browser through exactly three tools: +export const FACADE_AGENT_INSTRUCTIONS = `Browser tool surface: Stagehand Playwright facade. +You control one persistent browser through exactly three tools: +- run: execute JavaScript against an initialized Playwright page, context, and browser (page.goto, page.locator(selector).click()/fill(), page.getByRole(...), page.evaluate(...), and the supported Playwright-shaped API). Use await directly and return JSON-serializable values so you can inspect progress. Alternatively, pass snapshot actions. +- snapshot: inspect the active page's accessibility tree and hydrate bracketed element IDs for run actions. +- screenshot: inspect the rendered page visually. + +Pass run exactly one of code or actions; every action uses "op" and "id", never "kind" or "ref". Snapshot IDs are valid only for the latest snapshot of the active page; snapshot again after navigation or stale IDs. The first browser action should usually be: await page.goto(url, { waitUntil: 'domcontentloaded' }). Do not launch another browser or create a separate browser process.`; + +/** + * Agent instructions of the pre-Playwright-idiom facade surface, kept + * byte-identical for `--surface=legacy` hosts. + */ +export const LEGACY_FACADE_AGENT_INSTRUCTIONS = `You control one persistent browser through exactly three tools: - snapshot: inspect the active page and hydrate bracketed element IDs. - run: provide either snapshot actions or JavaScript using the Playwright-shaped page API. - screenshot: inspect the rendered page visually. @@ -140,6 +230,31 @@ export function staleSnapshotIdError(id: string): string { return STALE_SNAPSHOT_ID_ERROR.replace("${id}", id); } +/** + * Prefix of the terminal error every facade tool returns once the browser + * session is gone. Harnesses match on it to tell consequences from agent errors. + */ +export const BROWSER_SESSION_LOST_ERROR_PREFIX = "Browser session lost ("; +export const BROWSER_SESSION_LOST_ERROR = + "Browser session lost (${cause}). The task cannot continue; report your final result now."; +/** Stderr telemetry line the stdio server emits once when the session is lost. */ +export const SESSION_LOST_TELEMETRY_PREFIX = "stagehand_facade_session_lost "; + +export function browserSessionLostError(cause: string): string { + return BROWSER_SESSION_LOST_ERROR.replace("${cause}", cause); +} + +export function isBrowserSessionLostError(message: string): boolean { + return message.startsWith(BROWSER_SESSION_LOST_ERROR_PREFIX); +} + +export type FacadeSessionLoss = { + cause: string; + /** Tool call that first observed the loss. */ + tool: string; + at: string; +}; + export const RefActionSchema = z.discriminatedUnion("op", [ z.strictObject({ op: z.literal("click"), id: z.string().min(1) }), z.strictObject({ op: z.literal("hover"), id: z.string().min(1) }), diff --git a/packages/integrations/core/src/facade/index.ts b/packages/integrations/core/src/facade/index.ts index 6bb6f38877..c78f8df1fb 100644 --- a/packages/integrations/core/src/facade/index.ts +++ b/packages/integrations/core/src/facade/index.ts @@ -1,9 +1,20 @@ export { FACADE_TOOLS, + FACADE_LEGACY_TOOLS, FACADE_AGENT_INSTRUCTIONS, + LEGACY_FACADE_AGENT_INSTRUCTIONS, RUN_TOOL_DESCRIPTION, + LEGACY_RUN_TOOL_DESCRIPTION, + facadeSurfaceFromArgs, + facadeToolsForSurface, + facadeToolsFor, + facadeAgentInstructions, + explicitSnapshotActionsEnabled, + SNAPSHOT_ACTIONS_WORKFLOW, + type FacadeSurface, SNAPSHOT_TOOL_DESCRIPTION, SCREENSHOT_TOOL_DESCRIPTION, + SESSION_INFO_TOOL_NAME, RUN_INPUT_SCHEMA, SNAPSHOT_INPUT_SCHEMA, SCREENSHOT_INPUT_SCHEMA, @@ -11,6 +22,12 @@ export { NAVIGATED_SNAPSHOT_ERROR, STALE_SNAPSHOT_ID_ERROR, staleSnapshotIdError, + BROWSER_SESSION_LOST_ERROR, + BROWSER_SESSION_LOST_ERROR_PREFIX, + SESSION_LOST_TELEMETRY_PREFIX, + browserSessionLostError, + isBrowserSessionLostError, + type FacadeSessionLoss, RefActionSchema, CodeModeRunInputSchema, SnapshotInputSchema, @@ -18,8 +35,14 @@ export { type RefAction, type CodeModeRunInput, } from "./contract.js"; -export { StagehandFacadeTools } from "./tools.js"; export { + StagehandFacadeTools, + type StagehandFacadeRunReport, + type StagehandFacadeToolsOptions, +} from "./tools.js"; +export { + DEFAULT_BROWSERBASE_SESSION_TIMEOUT_SECONDS, + MAX_BROWSERBASE_SESSION_TIMEOUT_SECONDS, StagehandFacadeConfigError, stagehandFacadeConfigFromEnv, type StagehandFacadeConfig, diff --git a/packages/integrations/core/src/facade/runtime.ts b/packages/integrations/core/src/facade/runtime.ts index 03242d1cab..ef6d171051 100644 --- a/packages/integrations/core/src/facade/runtime.ts +++ b/packages/integrations/core/src/facade/runtime.ts @@ -118,8 +118,14 @@ export type PlaywrightCompatRuntime = { * Keep this function self-contained: its source is serialized into the * extension service worker with Function#toString. */ +export type PlaywrightCompatRuntimeOptions = { + /** Host-owned pages excluded from the agent context and page events. */ + hiddenPageIds?: string[]; +}; + export async function createPlaywrightCompatRuntime( stagehand: BatchStagehandRuntime, + options: PlaywrightCompatRuntimeOptions = {}, ): Promise { type CompatStats = PlaywrightCompatTelemetry; type QueryResult = { @@ -623,6 +629,11 @@ export async function createPlaywrightCompatRuntime( const rawContext = stagehand.context; type PageKey = string | RawPage; const pageKey = (page: RawPage): PageKey => page.pageId ?? page; + const hiddenPageIds = new Set(options.hiddenPageIds ?? []); + const visibleRawPages = async (): Promise => + (await rawContext.pages()).filter( + (page) => page.pageId === undefined || !hiddenPageIds.has(page.pageId), + ); const compatPages = new Map(); const closedPages = new Set(); let closeRequested = false; @@ -1231,7 +1242,7 @@ export async function createPlaywrightCompatRuntime( }, refreshUrl: async () => { state.cachedUrl = await page.url(); - for (const candidate of await rawContext.pages()) await createPage(candidate); + for (const candidate of await visibleRawPages()) await createPage(candidate); }, }; const root = (): CompatLocator => locatorProxy(new CompatLocator([], state)); @@ -1697,7 +1708,7 @@ export async function createPlaywrightCompatRuntime( waitForTimeout: async (ms: number) => { record("calls", "page.waitForTimeout"); await page.waitForTimeout(ms); - for (const candidate of await rawContext.pages()) await createPage(candidate); + for (const candidate of await visibleRawPages()) await createPage(candidate); }, waitForLoadState: (state = "load", options: { timeout?: number } = {}) => { record("calls", "page.waitForLoadState"); @@ -1910,7 +1921,7 @@ export async function createPlaywrightCompatRuntime( const initialPage = await createPage(stagehand.page); const contextRequest = (initialPage as { request: unknown }).request; - const initialRawPages = await rawContext.pages(); + const initialRawPages = await visibleRawPages(); for (const page of initialRawPages) await createPage(page); waitForNewPage = async (options: { timeout?: number } = {}): Promise => { @@ -1918,7 +1929,7 @@ export async function createPlaywrightCompatRuntime( const timeout = options.timeout ?? 30_000; const deadline = Date.now() + timeout; do { - for (const candidate of await rawContext.pages()) { + for (const candidate of await visibleRawPages()) { const key = pageKey(candidate); if (!existing.has(key)) return await createPage(candidate); } diff --git a/packages/integrations/core/src/facade/stdio-server.ts b/packages/integrations/core/src/facade/stdio-server.ts index 53b1bc5bac..cec3f2c1bf 100644 --- a/packages/integrations/core/src/facade/stdio-server.ts +++ b/packages/integrations/core/src/facade/stdio-server.ts @@ -14,9 +14,9 @@ import { sanitizeErrorMessage } from "../harness/redact.js"; import { stagehandFacadeConfigFromEnv } from "./config.js"; import { CodeModeRunInputSchema, - FACADE_TOOLS, - SCREENSHOT_TOOL_DESCRIPTION, - SNAPSHOT_TOOL_DESCRIPTION, + facadeSurfaceFromArgs, + facadeToolsFor, + SESSION_INFO_TOOL_NAME, ScreenshotInputSchema, SnapshotInputSchema, } from "./contract.js"; @@ -34,27 +34,28 @@ type FacadeResources = { const server = new McpServer({ name: "stagehand-facade", version: "4.0.0" }); const screenshotBase64Budget = screenshotBase64BudgetFromArgs(process.argv.slice(2)); +const facadeTools = facadeToolsFor(facadeSurfaceFromArgs(process.argv.slice(2))); let resourcesPromise: Promise | undefined; let closing = false; server.registerTool( "run", - { description: FACADE_TOOLS[0].description, inputSchema: CodeModeRunInputSchema }, + { description: facadeTools[0].description, inputSchema: CodeModeRunInputSchema }, async () => ({ content: [] }), ); server.registerTool( "snapshot", - { description: SNAPSHOT_TOOL_DESCRIPTION, inputSchema: SnapshotInputSchema }, + { description: facadeTools[1].description, inputSchema: SnapshotInputSchema }, async () => ({ content: [] }), ); server.registerTool( "screenshot", - { description: SCREENSHOT_TOOL_DESCRIPTION, inputSchema: ScreenshotInputSchema }, + { description: facadeTools[2].description, inputSchema: ScreenshotInputSchema }, async () => ({ content: [] }), ); server.server.removeRequestHandler("tools/list"); -server.server.setRequestHandler(ListToolsRequestSchema, () => ({ tools: [...FACADE_TOOLS] })); +server.server.setRequestHandler(ListToolsRequestSchema, () => ({ tools: [...facadeTools] })); server.server.removeRequestHandler("tools/call"); server.server.setRequestHandler(CallToolRequestSchema, async (request) => { try { @@ -101,6 +102,18 @@ server.server.setRequestHandler(CallToolRequestSchema, async (request) => { ], }; } + case SESSION_INFO_TOOL_NAME: { + // Runner-side only (absent from tools/list): launches the browser if + // needed and reports where it lives so the harness can log the + // Browserbase session URL before the agent's first call. + const browser = (await ensureResources()).browser; + return textResult( + JSON.stringify({ + provider: browser.provider, + ...(browser.sessionId && { sessionId: browser.sessionId }), + }), + ); + } default: throw new Error(`Unknown tool: ${request.params.name}`); } @@ -125,7 +138,11 @@ async function createResources(): Promise { : await localBrowser.launch(config.browser.launchOptions); try { const stagehand = await Stagehand.create({ browser, ...config.stagehand }); - return { browser, stagehand, tools: new StagehandFacadeTools(stagehand) }; + const tools = new StagehandFacadeTools(stagehand, { + onRunReport: (report) => + process.stderr.write(`stagehand_playwright_compat ${JSON.stringify(report)}\n`), + }); + return { browser, stagehand, tools }; } catch (error) { await browser.close().catch(() => undefined); throw error; @@ -162,10 +179,7 @@ async function shutdown(code: number): Promise { ...(resources ? [ { - close: async () => { - await resources.stagehand.close().catch(() => undefined); - await resources.browser.close(); - }, + close: () => resources.tools.close(), }, ] : []), diff --git a/packages/integrations/core/src/facade/tools.ts b/packages/integrations/core/src/facade/tools.ts index e291407d3f..30257430eb 100644 --- a/packages/integrations/core/src/facade/tools.ts +++ b/packages/integrations/core/src/facade/tools.ts @@ -1,4 +1,8 @@ +import fsp from "node:fs/promises"; +import { constants } from "node:fs"; +import path from "node:path"; import type { ExperimentalBatchCallback, Page, Stagehand } from "@browserbasehq/stagehand"; +import { sanitizeErrorMessage } from "../harness/redact.js"; import { NAVIGATED_SNAPSHOT_ERROR, NO_HYDRATED_SNAPSHOT_ERROR, @@ -6,17 +10,50 @@ import { staleSnapshotIdError, type RefAction, } from "./contract.js"; -import { createPlaywrightCompatRuntime } from "./runtime.js"; +import { createPlaywrightCompatRuntime, type PlaywrightCompatTelemetry } from "./runtime.js"; type SnapshotState = { url: string; xpathById: Record }; type HydratedAction = RefAction & { selector: string }; type ActionResult = { completed: number }; +type ScreenshotArtifact = { path: string; base64: string }; type RunEnvelope = { __stagehandPlaywrightCompat: true; value: unknown; executionError?: { name: string; message: string; stack?: string }; + telemetry: PlaywrightCompatTelemetry; + artifacts: ScreenshotArtifact[]; + closeRequested: boolean; + batchRuntimeMs: number; }; +export type StagehandFacadeRunReport = { + telemetry: PlaywrightCompatTelemetry; + /** Wall-clock time of the whole experimentalBatch round trip. */ + batchRoundTripMs: number; + /** Time the agent's code spent executing inside the batch. */ + batchRuntimeMs: number; + closeRequested: boolean; +}; + +export type StagehandFacadeToolsOptions = { + /** Owned directory for screenshot artifacts; paths must stay within it. Defaults to process.cwd(). */ + artifactRoot?: string; + /** Observes every completed `run` batch (including ones whose code threw). */ + onRunReport?: (report: StagehandFacadeRunReport) => void; + /** + * Keep a hidden about:blank tab open for the whole session (default true). + * Chrome exits when its last tab closes, so a renderer crash on the agent's + * only tab ("Render process gone" on heavy retail pages) used to end the + * Browserbase session and turn one bad page into a terminal + * "Browser session lost". With the keeper, the browser survives and the next + * call gets a fresh page instead. The keeper is invisible to agent code + * (`context.pages()` / `waitForEvent("page")` never list it). + */ + keeperPage?: boolean; +}; + +const RUN_BATCH_TIMEOUT_MS = 60_000; + export type StagehandFacadeScreenshot = { data: string; mimeType: "image/png" | "image/jpeg"; @@ -49,17 +86,21 @@ const actionRunner = new AsyncFunction( ACTION_RUNNER_SOURCE, ) as ExperimentalBatchCallback<{ actions: HydratedAction[] }, ActionResult>; +/** URL of the hidden keeper tab (see StagehandFacadeToolsOptions.keeperPage). */ +export const FACADE_KEEPER_PAGE_URL = "about:blank"; + const FACADE_PRELUDE = `"use strict"; const __stagehandCompatIdentity = (target) => target; for (let index = 0; index <= 32; index += 1) { globalThis[index === 0 ? "__name" : "__name" + index] = __stagehandCompatIdentity; } const createRuntime = ${createPlaywrightCompatRuntime.toString()}; -const runtime = await createRuntime(batchStagehand); +const runtime = await createRuntime(batchStagehand, { hiddenPageIds: input.hiddenPageIds ?? [] }); const page = runtime.page; const context = runtime.context; const browser = runtime.browser; const console = globalThis.console; +const __stagehandBatchStartedAt = performance.now(); let value; let executionError; try { @@ -78,30 +119,61 @@ return { __stagehandPlaywrightCompat: true, value, executionError, + telemetry: runtime.telemetry(), + artifacts: runtime.artifacts(), + closeRequested: runtime.closeRequested(), + batchRuntimeMs: performance.now() - __stagehandBatchStartedAt, };`; +type RunInput = { hiddenPageIds?: string[] }; + export class StagehandFacadeTools { private readonly snapshotsByPage = new Map(); private queue: Promise = Promise.resolve(); + private keeper: Promise | undefined; + private closed = false; + private closePromise: Promise | undefined; + constructor( + private readonly stagehand: Stagehand, + private readonly options: StagehandFacadeToolsOptions = {}, + ) {} - constructor(private readonly stagehand: Stagehand) {} + /** Closes both the client and its owned browser, including keep-alive sessions. */ + close(): Promise { + this.closed = true; + return (this.closePromise ??= (async () => { + const failures: unknown[] = []; + for (const close of [() => this.stagehand.close(), () => this.stagehand.browser.close()]) { + try { + await close(); + } catch (error) { + failures.push(error); + } + } + if (failures.length) { + throw new Error("Failed to close the Stagehand facade browser.", { + cause: new AggregateError(failures, "Facade browser cleanup failures"), + }); + } + })()); + } snapshot(options: { includeIframes?: boolean } = {}): Promise { - return this.enqueue(() => this.snapshotNow(options)); + return this.enqueue("snapshot", () => this.snapshotNow(options)); } screenshot( options: { fullPage?: boolean; type?: "png" | "jpeg"; quality?: number } = {}, ): Promise { - return this.enqueue(() => this.screenshotNow(options)); + return this.enqueue("screenshot", () => this.screenshotNow(options)); } runActions(actions: RefAction[]): Promise<{ completed: number; url: string }> { - return this.enqueue(() => this.runActionsNow(actions)); + return this.enqueue("run", () => this.runActionsNow(actions)); } run(code: string): Promise { - return this.enqueue(() => this.runNow(code)); + return this.enqueue("run", () => this.runNow(code)); } private async snapshotNow(options: { includeIframes?: boolean }): Promise { @@ -147,7 +219,7 @@ export class StagehandFacadeTools { } const hydrated = parsed.map((action) => { - const xpath = trimTrailingTextNode(snapshot.xpathById[action.id]); + const xpath = trimTrailingTextNode(resolveSnapshotXPath(snapshot.xpathById, action.id)); if (!xpath) throw new Error(staleSnapshotIdError(action.id)); return { ...action, selector: `xpath=${xpath}` }; }); @@ -165,29 +237,158 @@ export class StagehandFacadeTools { "batchStagehand", "input", FACADE_PRELUDE + code + FACADE_EPILOGUE, - ) as ExperimentalBatchCallback, RunEnvelope>; - const envelope = await this.stagehand.experimentalBatch( - callback, - {}, - { page, timeout: 60_000 }, - ); - if (envelope.executionError) { - const error = new Error(envelope.executionError.message); - error.name = envelope.executionError.name; - if (envelope.executionError.stack) error.stack = envelope.executionError.stack; - throw error; + ) as ExperimentalBatchCallback; + const startedAt = performance.now(); + const keeperPageId = await this.keeper; + const input: RunInput = keeperPageId ? { hiddenPageIds: [keeperPageId] } : {}; + const envelope = await this.runBatchWithActivePageFallback(callback, input, page); + try { + this.options.onRunReport?.({ + telemetry: envelope.telemetry, + batchRoundTripMs: performance.now() - startedAt, + batchRuntimeMs: envelope.batchRuntimeMs, + closeRequested: envelope.closeRequested, + }); + await this.writeScreenshotArtifacts(envelope.artifacts); + if (envelope.executionError) { + // Thrown by the agent's own code inside the browser, so its message can + // never be evidence about this process's connection to the browser. + const error = new Error(envelope.executionError.message) as Error & { + facadeExecutionError: true; + }; + error.name = envelope.executionError.name; + if (envelope.executionError.stack) error.stack = envelope.executionError.stack; + error.facadeExecutionError = true; + throw error; + } + return envelope.value; + } finally { + // The batch must finish before closing its own transport. This also + // honors close requests when agent code or artifact persistence failed. + if (envelope.closeRequested) await this.close(); + } + } + + /** + * The batch controller resolves its target page before invoking the + * callback, so when the active page vanished between activePage() and the + * batch (tab closed by the previous snippet) this retry cannot replay + * partially executed agent code. + */ + private async runBatchWithActivePageFallback( + callback: ExperimentalBatchCallback, + input: RunInput, + page: Page, + ): Promise { + try { + return await this.stagehand.experimentalBatch(callback, input, { + page, + timeout: RUN_BATCH_TIMEOUT_MS, + }); + } catch (error) { + if (!(error instanceof Error) || !/callback batch page was not found/iu.test(error.message)) { + throw error; + } + const fallback = await this.activePage(); + return await this.stagehand.experimentalBatch(callback, input, { + page: fallback, + timeout: RUN_BATCH_TIMEOUT_MS, + }); + } + } + + private async writeScreenshotArtifacts(artifacts: ScreenshotArtifact[]): Promise { + if (artifacts.length === 0) return; + const configuredRoot = path.resolve(this.options.artifactRoot ?? process.cwd()); + await fsp.mkdir(configuredRoot, { recursive: true }); + const root = await fsp.realpath(configuredRoot); + for (const artifact of artifacts) { + const target = path.resolve(configuredRoot, artifact.path); + const relative = path.relative(configuredRoot, target); + if ( + !relative || + relative === ".." || + relative.startsWith(`..${path.sep}`) || + path.isAbsolute(relative) + ) { + throw new Error("Screenshot artifact path must stay within artifactRoot."); + } + // Reject symlink traversal before creating nested directories or opening + // the output. O_NOFOLLOW also refuses an existing symlink at the file. + let directory = root; + const components = relative.split(path.sep); + for (const component of components.slice(0, -1)) { + directory = path.join(directory, component); + await fsp.mkdir(directory).catch((error: NodeJS.ErrnoException) => { + if (error.code !== "EEXIST") throw error; + }); + if (!(await fsp.lstat(directory)).isDirectory()) { + throw new Error("Screenshot artifact directory must not be a symlink."); + } + } + const file = await fsp.open( + path.join(directory, components.at(-1)!), + constants.O_WRONLY | constants.O_CREAT | constants.O_TRUNC | constants.O_NOFOLLOW, + 0o600, + ); + try { + await file.writeFile(Buffer.from(artifact.base64, "base64")); + } finally { + await file.close(); + } } - return envelope.value; } + /** + * The page the agent is working on. A closed or crashed tab leaves the + * context without an active page (or with only the keeper); the agent then + * gets a fresh blank page rather than a dead end, and the keeper stays hidden. + */ private async activePage(): Promise { - const page = await this.stagehand.browser.context.activePage(); - if (!page) throw new Error("Stagehand has no active page."); + const context = this.stagehand.browser.context; + const keeperPageId = await this.keeper; + const active = await context.activePage(); + if (active && active.pageId !== keeperPageId) return active; + const visible = (await context.pages()).filter((page) => page.pageId !== keeperPageId); + const page = visible[0] ?? (await context.newPage()); + await context.setActivePage(page); return page; } - private enqueue(operation: () => Promise): Promise { - const result = this.queue.then(operation, operation); + /** + * Opens the keeper once. A dead transport here is session loss like + * anywhere else; any other failure just means running without a keeper. + */ + private async ensureKeeperPage(): Promise { + this.keeper ??= this.openKeeperPage().catch((error: unknown) => { + if (sessionLossCause(error) !== undefined) throw error; + return undefined; + }); + try { + await this.keeper; + } catch (error) { + this.keeper = Promise.resolve(undefined); + throw error; + } + } + + private async openKeeperPage(): Promise { + if (this.options.keeperPage === false) return undefined; + const context = this.stagehand.browser.context; + const before = await context.activePage(); + const keeper = await context.newPage(FACADE_KEEPER_PAGE_URL); + // newPage activates the new tab; hand focus straight back. + if (before) await context.setActivePage(before); + return keeper.pageId; + } + + private enqueue(_tool: string, operation: () => Promise): Promise { + const execute = async (): Promise => { + if (this.closed) throw new Error("Stagehand facade browser is closed."); + await this.ensureKeeperPage(); + return operation(); + }; + const result = this.queue.then(execute, execute); this.queue = result.then( () => undefined, () => undefined, @@ -196,6 +397,62 @@ export class StagehandFacadeTools { } } +/** + * Maps an error to the reason the browser session is unusable, or undefined + * when it is an ordinary tool failure the agent can act on. A batch that hit + * its executor-side timeout is ordinary: the executor answered. + */ +function sessionLossCause(error: unknown): string | undefined { + if (!(error instanceof Error)) return undefined; + if ((error as { facadeExecutionError?: boolean }).facadeExecutionError) return undefined; + switch (error.name) { + case "StagehandBatchTimeoutError": { + const { clientTimeout } = error as Error & { clientTimeout?: number }; + return typeof clientTimeout === "number" + ? `batch received no response within ${clientTimeout}ms` + : "batch received no response before its client deadline"; + } + // FacadeDeadlineError is intentionally NOT terminal here: a single capture + // deadline is handled as recoverable in enqueue(), which escalates to + // session-loss only after repeated consecutive timeouts. + case "CDPConnectionClosedError": + return cdpSessionLossCause(error); + } + if (/\bCDP connection closed\b/u.test(error.message)) return cdpSessionLossCause(error); + if (/\bRPC client is closed\b/u.test(error.message)) return "RPC client closed"; + return undefined; +} + +function cdpSessionLossCause(error: Error): string { + const messages = [error.message]; + const seen = new Set([error]); + let cause = error.cause; + // An error before the WebSocket close event carries its diagnostics in cause, + // while a close event carries them in the outer message. Preserve both paths. + while (cause instanceof Error && !seen.has(cause)) { + seen.add(cause); + const code = (cause as Error & { code?: unknown }).code; + const label = + typeof code === "string" || typeof code === "number" ? `${cause.name} [${code}]` : cause.name; + messages.push(cause.message ? `${label}: ${cause.message}` : label); + cause = cause.cause; + } + return sanitizeErrorMessage(messages.join("; caused by ")); +} + function trimTrailingTextNode(path: string | undefined): string | undefined { return path?.replace(/\/text\(\)(\[\d+\])?$/iu, ""); } + +/** + * Snapshot IDs are `-` (e.g. "0-7812"). Models + * regularly copy only the backend id; accept that when it is unambiguous. + */ +function resolveSnapshotXPath(xpathById: Record, id: string): string | undefined { + const exact = xpathById[id]; + if (exact !== undefined) return exact; + if (id.includes("-")) return undefined; + const suffix = `-${id}`; + const matches = Object.keys(xpathById).filter((key) => key.endsWith(suffix)); + return matches.length === 1 ? xpathById[matches[0]!] : undefined; +} diff --git a/packages/integrations/core/src/harness/env.ts b/packages/integrations/core/src/harness/env.ts index 43069f433a..d3ef30f696 100644 --- a/packages/integrations/core/src/harness/env.ts +++ b/packages/integrations/core/src/harness/env.ts @@ -1,7 +1,9 @@ /** Only STAGEHAND_* and BROWSERBASE_* host env vars cross into harness processes. */ -export function buildAllowlistedEnv(): Record { +export function buildAllowlistedEnv( + source: NodeJS.ProcessEnv = process.env, +): Record { const env: Record = {}; - for (const [key, value] of Object.entries(process.env)) { + for (const [key, value] of Object.entries(source)) { if (/^(STAGEHAND_|BROWSERBASE_)/.test(key) && value) { env[key] = value; } diff --git a/packages/integrations/core/tests/facade-contract.test.ts b/packages/integrations/core/tests/facade-contract.test.ts index 05db01c772..e1fe862876 100644 --- a/packages/integrations/core/tests/facade-contract.test.ts +++ b/packages/integrations/core/tests/facade-contract.test.ts @@ -1,12 +1,20 @@ import { describe, expect, it } from "vitest"; import { CodeModeRunInputSchema, + FACADE_AGENT_INSTRUCTIONS, + FACADE_LEGACY_TOOLS, FACADE_TOOLS, + facadeSurfaceFromArgs, + facadeToolsFor, + facadeToolsForSurface, + LEGACY_FACADE_AGENT_INSTRUCTIONS, NAVIGATED_SNAPSHOT_ERROR, NO_HYDRATED_SNAPSHOT_ERROR, STALE_SNAPSHOT_ID_ERROR, + explicitSnapshotActionsEnabled, + facadeAgentInstructions, } from "../src/facade/contract.js"; -import { stagehandFacadeConfigFromEnv } from "../src/facade/config.js"; +import { StagehandFacadeConfigError, stagehandFacadeConfigFromEnv } from "../src/facade/config.js"; describe("Stagehand facade contract", () => { it("defaults local browser launches to headed mode", () => { @@ -16,6 +24,88 @@ describe("Stagehand facade contract", () => { }); }); + it("ignores malformed Browserbase-only options in local mode", () => { + expect( + stagehandFacadeConfigFromEnv({ + STAGEHAND_BROWSER: "local", + STAGEHAND_BROWSERBASE_SESSION_TIMEOUT_SECONDS: "invalid", + STAGEHAND_BROWSERBASE_PROXIES: "invalid", + STAGEHAND_BROWSERBASE_VERIFIED: "invalid", + }).browser.type, + ).toBe("local"); + }); + + it("creates Browserbase sessions with an explicit timeout and keep-alive", () => { + const env = { BROWSERBASE_API_KEY: "bb-key", BROWSERBASE_PROJECT_ID: "proj" }; + expect(stagehandFacadeConfigFromEnv(env).browser).toStrictEqual({ + type: "browserbase", + launchOptions: { apiKey: "bb-key", projectId: "proj", timeout: 3600, keepAlive: true }, + }); + expect( + stagehandFacadeConfigFromEnv({ + ...env, + STAGEHAND_BROWSERBASE_SESSION_TIMEOUT_SECONDS: "21600", + }).browser.launchOptions, + ).toMatchObject({ timeout: 21_600 }); + for (const invalid of ["0", "1", "59", "-1", "90.5", "soon", "21601"]) { + expect(() => + stagehandFacadeConfigFromEnv({ + ...env, + STAGEHAND_BROWSERBASE_SESSION_TIMEOUT_SECONDS: invalid, + }), + ).toThrow(StagehandFacadeConfigError); + } + }); + + it("passes proxies/verified through to the Browserbase session when configured", () => { + const env = { BROWSERBASE_API_KEY: "bb-key" }; + expect(stagehandFacadeConfigFromEnv(env).browser.launchOptions).not.toHaveProperty("proxies"); + expect( + stagehandFacadeConfigFromEnv({ + ...env, + STAGEHAND_BROWSERBASE_PROXIES: "1", + STAGEHAND_BROWSERBASE_VERIFIED: "true", + }).browser.launchOptions, + ).toMatchObject({ proxies: true, browserSettings: { verified: true } }); + expect( + stagehandFacadeConfigFromEnv({ ...env, STAGEHAND_BROWSERBASE_PROXIES: "off" }).browser + .launchOptions, + ).toMatchObject({ proxies: false }); + expect(() => + stagehandFacadeConfigFromEnv({ ...env, STAGEHAND_BROWSERBASE_VERIFIED: "maybe" }), + ).toThrow(/STAGEHAND_BROWSERBASE_VERIFIED must be a boolean/); + }); + + it("uses a host-provided extension id instead of uploading per session", () => { + const env = { BROWSERBASE_API_KEY: "bb-key" }; + expect(stagehandFacadeConfigFromEnv(env).browser.launchOptions).not.toHaveProperty( + "extensionId", + ); + expect( + stagehandFacadeConfigFromEnv({ ...env, STAGEHAND_BROWSERBASE_EXTENSION_ID: " ext_shared " }) + .browser.launchOptions, + ).toMatchObject({ extensionId: "ext_shared" }); + }); + + it("adds snapshot/screenshot routing guidance only when EXPLICIT_SNAPSHOT_ACTIONS is on", () => { + expect(explicitSnapshotActionsEnabled({})).toBe(false); + expect(explicitSnapshotActionsEnabled({ EXPLICIT_SNAPSHOT_ACTIONS: "1" })).toBe(true); + expect(facadeAgentInstructions({})).toBe(FACADE_AGENT_INSTRUCTIONS); + const explicit = facadeAgentInstructions({ EXPLICIT_SNAPSHOT_ACTIONS: "1" }); + expect(explicit.startsWith(FACADE_AGENT_INSTRUCTIONS)).toBe(true); + expect(explicit).toContain("act by snapshot ID"); + expect(explicit).toContain("Call screenshot before declaring success"); + expect(facadeToolsFor("playwright", {})).toBe(FACADE_TOOLS); + const tools = facadeToolsFor("playwright", { EXPLICIT_SNAPSHOT_ACTIONS: "true" }); + expect(tools.map((t) => t.name)).toEqual(["run", "snapshot", "screenshot"]); + expect(tools[0].description).toContain('Prefer "actions" with snapshot IDs'); + expect(tools[1].description).toContain("before acting by ID"); + expect(tools[2].description).toContain("before reporting done"); + // Same input schemas — prompt-only variant, not a new surface. + expect(tools.map((t) => t.inputSchema)).toEqual(FACADE_TOOLS.map((t) => t.inputSchema)); + expect(facadeToolsFor("legacy", { EXPLICIT_SNAPSHOT_ACTIONS: "1" })).toBe(FACADE_LEGACY_TOOLS); + }); + it("pins the three tool names and descriptions", () => { expect(FACADE_TOOLS.map((tool) => tool.name)).toStrictEqual(["run", "snapshot", "screenshot"]); expect(FACADE_TOOLS[0].description).toContain("Browse and automate websites"); @@ -33,6 +123,45 @@ describe("Stagehand facade contract", () => { expect(FACADE_TOOLS[2].description).toContain('{"type":"jpeg","quality":40,"fullPage":false}'); }); + it("advertises the Playwright idiom and no Stagehand AI methods", () => { + expect(FACADE_TOOLS[0].description).toContain("page, context, and browser in scope"); + expect(FACADE_TOOLS[0].description).toContain("Playwright-shaped API"); + expect(FACADE_AGENT_INSTRUCTIONS).toContain("Playwright page, context, and browser"); + expect(FACADE_AGENT_INSTRUCTIONS).toContain("page.getByRole("); + for (const text of [FACADE_TOOLS[0].description, FACADE_AGENT_INSTRUCTIONS]) { + expect(text).not.toMatch(/\b(act|extract|observe)\(/u); + expect(text).not.toContain("stagehand."); + } + }); + + it("keeps the legacy surface byte-identical except for the run description", () => { + expect(FACADE_LEGACY_TOOLS.map((tool) => tool.name)).toStrictEqual([ + "run", + "snapshot", + "screenshot", + ]); + expect(FACADE_LEGACY_TOOLS[0].description).toBe( + 'Browse and automate websites in the persistent Stagehand browser. Navigate with JavaScript such as await page.goto("https://example.com"); there is no separate navigate or start tool. Execute either a JavaScript workflow against the Stagehand Playwright facade or a batch of actions using IDs from the latest snapshot. Provide exactly one of code or actions. Each action must use "op" (never "kind") and "id" (never "ref"). Copy the bracketed snapshot ID as a string. Examples: {"actions":[{"op":"click","id":"1-42"}]}, {"actions":[{"op":"fill","id":"2-14","value":"Miami"}]}, {"actions":[{"op":"select","id":"3-9","values":"Lowest price"}]}.', + ); + expect(FACADE_LEGACY_TOOLS[0].inputSchema).toBe(FACADE_TOOLS[0].inputSchema); + expect(FACADE_LEGACY_TOOLS[1]).toBe(FACADE_TOOLS[1]); + expect(FACADE_LEGACY_TOOLS[2]).toBe(FACADE_TOOLS[2]); + expect(LEGACY_FACADE_AGENT_INSTRUCTIONS).toContain( + "Use snapshot actions for simple interactions", + ); + expect(LEGACY_FACADE_AGENT_INSTRUCTIONS).not.toBe(FACADE_AGENT_INSTRUCTIONS); + }); + + it("selects the surface from --surface", () => { + expect(facadeSurfaceFromArgs([])).toBe("playwright"); + expect(facadeSurfaceFromArgs(["--max-screenshot-base64-bytes=4096"])).toBe("playwright"); + expect(facadeSurfaceFromArgs(["--surface=legacy"])).toBe("legacy"); + expect(facadeSurfaceFromArgs(["--surface=playwright"])).toBe("playwright"); + expect(() => facadeSurfaceFromArgs(["--surface=codemode"])).toThrow("--surface="); + expect(facadeToolsForSurface("legacy")).toBe(FACADE_LEGACY_TOOLS); + expect(facadeToolsForSurface("playwright")).toBe(FACADE_TOOLS); + }); + it("pins snapshot error punctuation", () => { expect(NO_HYDRATED_SNAPSHOT_ERROR).toBe( "No hydrated snapshot exists for the active page; call snapshot first.", diff --git a/packages/integrations/core/tests/facade-server.test.ts b/packages/integrations/core/tests/facade-server.test.ts index e7a83613d6..b606384276 100644 --- a/packages/integrations/core/tests/facade-server.test.ts +++ b/packages/integrations/core/tests/facade-server.test.ts @@ -3,7 +3,7 @@ import type { Stream } from "node:stream"; import { Client } from "@modelcontextprotocol/sdk/client/index.js"; import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js"; import { afterEach, beforeEach, describe, expect, it } from "vitest"; -import { FACADE_TOOLS } from "../src/facade/contract.js"; +import { FACADE_LEGACY_TOOLS, FACADE_TOOLS } from "../src/facade/contract.js"; const entrypoint = fileURLToPath(new URL("../dist/facade/stdio-server.mjs", import.meta.url)); const readyMessage = "Stagehand facade MCP host listening on stdio"; @@ -66,6 +66,28 @@ describe("built Stagehand facade stdio server", () => { }); }); +describe("built Stagehand facade stdio server with --surface=legacy", () => { + it("lists the legacy run description", async () => { + const transport = new StdioClientTransport({ + command: process.execPath, + args: [entrypoint, "--surface=legacy"], + env: { PATH: process.env.PATH ?? "", STAGEHAND_BROWSER: "invalid" }, + stderr: "pipe", + }); + if (!transport.stderr) throw new Error("stdio transport did not expose stderr"); + const ready = waitForOutput(transport.stderr, readyMessage); + const client = new Client({ name: "stagehand-facade-test", version: "1.0.0" }); + try { + await Promise.all([client.connect(transport), ready]); + const result = await client.listTools(); + expect(result.tools).toStrictEqual([...FACADE_LEGACY_TOOLS]); + expect(result.tools[0].description).not.toBe(FACADE_TOOLS[0].description); + } finally { + await client.close(); + } + }); +}); + function textContent(result: Awaited>): string { const block = result.content[0]; return block && block.type === "text" ? block.text : ""; diff --git a/packages/integrations/core/tests/facade-tools.test.ts b/packages/integrations/core/tests/facade-tools.test.ts new file mode 100644 index 0000000000..eb66be00ed --- /dev/null +++ b/packages/integrations/core/tests/facade-tools.test.ts @@ -0,0 +1,392 @@ +import fsp from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { type Stagehand } from "@browserbasehq/stagehand"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { StagehandFacadeTools, type StagehandFacadeRunReport } from "../src/facade/tools.js"; + +type FakePage = ReturnType; + +type FakeRawLocator = { + selector: string; + nthIndex?: number; + click: ReturnType; + hover: ReturnType; + fill: ReturnType; + type: ReturnType; + selectOption: ReturnType; + setInputFiles: ReturnType; + count: ReturnType; + nth: (index: number) => FakeRawLocator; + isVisible: ReturnType; + isChecked: ReturnType; + inputValue: ReturnType; + innerText: ReturnType; + innerHtml: ReturnType; + textContent: ReturnType; + scrollTo: ReturnType; +}; + +type FakeFrameWorld = { + /** Match count per raw selector (default 1). */ + counts: Record; + /** Error thrown by click() per raw selector. */ + clickErrors: Record; + /** Absolute XPath returned for the first-hop iframe host. */ + iframeXPath: string | null; + snapshot: { formattedTree: string; xpathMap: Record }; + locators: FakeRawLocator[]; +}; + +function createFakeWorld(): FakeFrameWorld { + return { + counts: {}, + clickErrors: {}, + iframeXPath: "/html[1]/body[1]/div[2]/iframe[1]", + snapshot: { formattedTree: "", xpathMap: {} }, + locators: [], + }; +} + +function createFakePage(initialUrl = "about:blank", world: FakeFrameWorld = createFakeWorld()) { + let currentUrl = initialUrl; + const makeLocator = (selector: string, nthIndex?: number): FakeRawLocator => { + const locator: FakeRawLocator = { + selector, + ...(nthIndex === undefined ? {} : { nthIndex }), + click: vi.fn(async () => { + const error = world.clickErrors[selector]; + if (error) throw error; + }), + hover: vi.fn(async () => undefined), + fill: vi.fn(async () => undefined), + type: vi.fn(async () => undefined), + selectOption: vi.fn(async (values: string | string[]) => + Array.isArray(values) ? values : [values], + ), + setInputFiles: vi.fn(async () => undefined), + count: vi.fn(async () => world.counts[selector] ?? 1), + nth: (index: number) => makeLocator(selector, index), + isVisible: vi.fn(async () => true), + isChecked: vi.fn(async () => false), + inputValue: vi.fn(async () => "value"), + innerText: vi.fn(async () => "inner"), + innerHtml: vi.fn(async () => "inner"), + textContent: vi.fn(async () => "text"), + scrollTo: vi.fn(async () => undefined), + }; + world.locators.push(locator); + return locator; + }; + return { + pageId: "page-1", + locator: vi.fn((selector: string) => makeLocator(selector)), + keyPress: vi.fn(async () => undefined), + waitForTimeout: vi.fn(async () => undefined), + goto: vi.fn(async (url: string) => { + currentUrl = url; + return null; + }), + url: vi.fn(async () => currentUrl), + title: vi.fn(async () => "Example Domain"), + evaluate: vi.fn(async (expression: unknown) => { + if (typeof expression === "string" && expression.includes("innerWidth")) { + return { width: 1280, height: 720 }; + } + if (typeof expression === "string" && expression.includes("previousElementSibling")) { + return world.iframeXPath; + } + return undefined; + }), + screenshot: vi.fn(async () => Uint8Array.from([0x89, 0x50, 0x4e, 0x47])), + close: vi.fn(async () => undefined), + on: vi.fn(async () => ({ unsubscribe: async () => undefined })), + onCDP: vi.fn(async () => ({ unsubscribe: async () => undefined })), + sendCDP: vi.fn(async () => ({})), + snapshot: vi.fn(async () => world.snapshot), + }; +} + +/** + * Stands in for Stagehand: experimentalBatch() invokes the callback in-process + * with a batch-shaped { page, context } so the Playwright compat runtime runs + * exactly as it would inside the extension, minus the browser. + */ +function createFakeStagehand(page: FakePage) { + // The facade opens a hidden about:blank keeper tab on first use; give it its + // own identity so tests can tell it apart from the agent's page. + const keeper = { ...createFakePage("about:blank"), pageId: "keeper-page" }; + const context = { + activePage: vi.fn(async (): Promise => page), + newPage: vi.fn(async (url?: string) => (url === "about:blank" ? keeper : page)), + pages: vi.fn(async () => [page, keeper]), + setActivePage: vi.fn(async () => undefined), + }; + const experimentalBatch = vi.fn( + async ( + callback: (stagehand: unknown, input: unknown) => Promise, + input: unknown, + _options: { page?: unknown; timeout: number }, + ) => callback({ page, context }, input), + ); + return { + stagehand: { + browser: { context, close: vi.fn(async () => undefined) }, + close: vi.fn(async () => undefined), + experimentalBatch, + } as unknown as Stagehand, + context, + experimentalBatch, + keeper, + }; +} + +describe("StagehandFacadeTools.run (Playwright batch surface)", () => { + const tempDirs: string[] = []; + + afterEach(async () => { + await Promise.all(tempDirs.map((dir) => fsp.rm(dir, { recursive: true, force: true }))); + tempDirs.length = 0; + }); + + it("executes Playwright-style code with page, context, and browser in scope", async () => { + const page = createFakePage(); + const { stagehand, experimentalBatch } = createFakeStagehand(page); + const reports: StagehandFacadeRunReport[] = []; + const tools = new StagehandFacadeTools(stagehand, { onRunReport: (r) => reports.push(r) }); + + const result = await tools.run(` + await page.goto("https://example.com", { waitUntil: "domcontentloaded" }); + return { + url: page.url(), + title: await page.title(), + contexts: browser.contexts().length, + sameContext: browser.contexts()[0] === context, + connected: browser.isConnected(), + }; + `); + + expect(result).toStrictEqual({ + url: "https://example.com", + title: "Example Domain", + contexts: 1, + sameContext: true, + connected: true, + }); + expect(page.goto).toHaveBeenCalledWith("https://example.com", { + waitUntil: "domcontentloaded", + }); + expect(experimentalBatch).toHaveBeenCalledTimes(1); + expect(experimentalBatch.mock.calls[0][2]).toStrictEqual({ page, timeout: 60_000 }); + expect(reports).toHaveLength(1); + expect(reports[0].telemetry.calls["page.goto"]).toBe(1); + expect(reports[0].closeRequested).toBe(false); + expect(reports[0].batchRuntimeMs).toBeGreaterThanOrEqual(0); + expect(reports[0].batchRoundTripMs).toBeGreaterThanOrEqual(reports[0].batchRuntimeMs); + }); + + it("does not expose Stagehand AI methods or the raw client to the snippet", async () => { + const { stagehand } = createFakeStagehand(createFakePage()); + const tools = new StagehandFacadeTools(stagehand); + + // Unknown members of the guarded page are throwing stubs, so a snippet + // that reaches for Stagehand's AI methods fails loudly instead of silently. + await expect(tools.run(`await page.act("click the button");`)).rejects.toThrow(/act/u); + await expect(tools.run(`return typeof stagehand;`)).resolves.toBe("undefined"); + }); + + it("rethrows snippet errors with their name and message", async () => { + const { stagehand } = createFakeStagehand(createFakePage()); + const reports: StagehandFacadeRunReport[] = []; + const tools = new StagehandFacadeTools(stagehand, { onRunReport: (r) => reports.push(r) }); + + await expect( + tools.run(`const error = new TypeError("boom"); throw error;`), + ).rejects.toMatchObject({ name: "TypeError", message: "boom" }); + // The report still fires so hosts can see telemetry for failed batches. + expect(reports).toHaveLength(1); + }); + + it("writes page.screenshot({ path }) artifacts under artifactRoot", async () => { + const page = createFakePage(); + const { stagehand } = createFakeStagehand(page); + const artifactRoot = await fsp.mkdtemp(path.join(os.tmpdir(), "facade-tools-")); + tempDirs.push(artifactRoot); + const tools = new StagehandFacadeTools(stagehand, { artifactRoot }); + + await tools.run(`await page.screenshot({ path: "shots/first.png" }); return "ok";`); + + const written = await fsp.readFile(path.join(artifactRoot, "shots", "first.png")); + expect([...written]).toStrictEqual([0x89, 0x50, 0x4e, 0x47]); + expect(page.screenshot).toHaveBeenCalledTimes(1); + }); + + it("reports browser.close() without closing the page inside the batch", async () => { + const page = createFakePage(); + const { stagehand } = createFakeStagehand(page); + const reports: StagehandFacadeRunReport[] = []; + const tools = new StagehandFacadeTools(stagehand, { onRunReport: (r) => reports.push(r) }); + + await expect(tools.run(`await browser.close(); return browser.isConnected();`)).resolves.toBe( + false, + ); + expect(page.close).not.toHaveBeenCalled(); + expect(reports[0].closeRequested).toBe(true); + expect(reports[0].telemetry.calls["browser.close"]).toBe(1); + }); + + it("closes owned resources only after the batch finishes and refuses later calls", async () => { + const { stagehand, experimentalBatch } = createFakeStagehand(createFakePage()); + const tools = new StagehandFacadeTools(stagehand); + await expect(tools.run(`await browser.close(); return "finished";`)).resolves.toBe("finished"); + expect(stagehand.close).toHaveBeenCalledOnce(); + expect(stagehand.browser.close).toHaveBeenCalledOnce(); + await tools.close(); + await expect(tools.snapshot()).rejects.toThrow("browser is closed"); + expect(experimentalBatch).toHaveBeenCalledOnce(); + expect(stagehand.browser.close).toHaveBeenCalledOnce(); + }); + + it("still releases the browser when client cleanup fails", async () => { + const { stagehand } = createFakeStagehand(createFakePage()); + vi.mocked(stagehand.close).mockRejectedValue(new Error("apiKey=private-key")); + const tools = new StagehandFacadeTools(stagehand); + await expect(tools.run(`await browser.close();`)).rejects.toThrow( + "Failed to close the Stagehand facade browser.", + ); + expect(stagehand.browser.close).toHaveBeenCalledOnce(); + await expect(tools.run("return 1")).rejects.toThrow("browser is closed"); + }); + + it("honors browser.close even when subsequent agent code throws", async () => { + const { stagehand } = createFakeStagehand(createFakePage()); + const tools = new StagehandFacadeTools(stagehand); + await expect( + tools.run(`await browser.close(); throw new Error("agent failure");`), + ).rejects.toThrow("agent failure"); + expect(stagehand.browser.close).toHaveBeenCalledOnce(); + }); + + it("confines artifacts across traversal, absolute paths, and symlinks", async () => { + const root = await fsp.mkdtemp(path.join(os.tmpdir(), "facade-paths-")); + tempDirs.push(root); + const artifactRoot = path.join(root, "artifacts"); + await fsp.mkdir(artifactRoot); + const outside = path.join(root, "outside"); + await fsp.mkdir(outside); + const existing = path.join(outside, "existing.png"); + await fsp.writeFile(existing, "preserve"); + await fsp.symlink(outside, path.join(artifactRoot, "linked-directory")); + await fsp.symlink(existing, path.join(artifactRoot, "linked-file.png")); + const { stagehand } = createFakeStagehand(createFakePage()); + const tools = new StagehandFacadeTools(stagehand, { artifactRoot }); + for (const requestedPath of [ + "../escape.png", + path.join(outside, "absolute.png"), + "linked-directory/created/sub.png", + "linked-file.png", + ]) { + await expect( + tools.run(`await page.screenshot({ path: ${JSON.stringify(requestedPath)} });`), + ).rejects.toThrow(); + } + expect(await fsp.readdir(outside)).toEqual(["existing.png"]); + expect(await fsp.readFile(existing, "utf8")).toBe("preserve"); + expect(await fsp.readdir(root)).toEqual(["artifacts", "outside"]); + }); + + it("retries without a page target when the batch page vanished", async () => { + const page = createFakePage(); + const { stagehand, context, experimentalBatch, keeper } = createFakeStagehand(page); + experimentalBatch.mockRejectedValueOnce( + new Error("Stagehand callback batch page was not found"), + ); + // activePage(): keeper setup, then the batch target, then (inside the + // fallback) the tab is gone and only the keeper is left. + context.activePage + .mockResolvedValueOnce(page) + .mockResolvedValueOnce(page) + .mockResolvedValueOnce(undefined); + context.pages.mockResolvedValueOnce([keeper]); + const tools = new StagehandFacadeTools(stagehand); + + await expect(tools.run(`return 42;`)).resolves.toBe(42); + // One newPage for the keeper, one for the replacement tab. + expect(context.newPage).toHaveBeenCalledTimes(2); + expect(context.newPage.mock.calls[1]).toStrictEqual([]); + expect(experimentalBatch).toHaveBeenCalledTimes(2); + expect(experimentalBatch.mock.calls[1][2]).toStrictEqual({ page, timeout: 60_000 }); + }); + + it("propagates other batch failures unchanged", async () => { + const { stagehand, experimentalBatch, context } = createFakeStagehand(createFakePage()); + experimentalBatch.mockRejectedValueOnce(new Error("extension disconnected")); + const tools = new StagehandFacadeTools(stagehand); + + await expect(tools.run(`return 1;`)).rejects.toThrow("extension disconnected"); + // Only the keeper tab was opened; no replacement page for an ordinary failure. + expect(context.newPage).toHaveBeenCalledTimes(1); + expect(context.newPage).toHaveBeenCalledWith("about:blank"); + }); +}); + +describe("StagehandFacadeTools keeper tab", () => { + it("opens a hidden about:blank keeper once and hands focus back to the agent's page", async () => { + const page = createFakePage("https://example.com"); + const { stagehand, context, keeper } = createFakeStagehand(page); + const tools = new StagehandFacadeTools(stagehand); + + await tools.run(`return 1;`); + await tools.run(`return 2;`); + + expect(context.newPage).toHaveBeenCalledTimes(1); + expect(context.newPage).toHaveBeenCalledWith("about:blank"); + expect(context.setActivePage).toHaveBeenCalledWith(page); + // The keeper never reaches agent code. + await expect(tools.run(`return context.pages().length;`)).resolves.toBe(1); + expect(keeper.pageId).toBe("keeper-page"); + }); + + it("keeps the keeper hidden after navigation and timeout rediscovery", async () => { + const { stagehand } = createFakeStagehand(createFakePage()); + const tools = new StagehandFacadeTools(stagehand); + await expect( + tools.run( + `await page.goto("https://example.com"); await page.waitForTimeout(1); return context.pages().length;`, + ), + ).resolves.toBe(1); + }); + + it("gives the agent a fresh page when its only tab is gone and the keeper remains", async () => { + const page = createFakePage("https://example.com"); + const { stagehand, context, keeper } = createFakeStagehand(page); + const tools = new StagehandFacadeTools(stagehand); + await tools.run(`return 1;`); + + // Renderer crash closed the agent's tab: only the keeper is left and active. + context.activePage.mockResolvedValueOnce(keeper); + context.pages.mockResolvedValueOnce([keeper]); + const replacement = createFakePage("about:blank"); + context.newPage.mockResolvedValueOnce(replacement); + + await tools.snapshot(); + expect(context.newPage).toHaveBeenLastCalledWith(); + expect(context.setActivePage).toHaveBeenLastCalledWith(replacement); + }); + + it("does not emit the hidden keeper as a new page event", async () => { + const page = createFakePage(); + const { stagehand } = createFakeStagehand(page); + const tools = new StagehandFacadeTools(stagehand); + await expect(tools.run('await context.waitForEvent("page", { timeout: 1 });')).rejects.toThrow( + /timed out|timeout/iu, + ); + }); + + it("can be disabled", async () => { + const { stagehand, context } = createFakeStagehand(createFakePage()); + const tools = new StagehandFacadeTools(stagehand, { keeperPage: false }); + await tools.run(`return 1;`); + expect(context.newPage).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/sdk-ts/src/browser/factories.ts b/packages/sdk-ts/src/browser/factories.ts index 362cbe6cd9..dabd03e348 100644 --- a/packages/sdk-ts/src/browser/factories.ts +++ b/packages/sdk-ts/src/browser/factories.ts @@ -319,7 +319,10 @@ async function connectBrowser(options: { }); } catch (error) { cdpClient?.close(); - if (ownsSource) { + // No browser handle escaped this failed launch. A newly created remote + // session remains ours to release, even if successful handles stay alive + // after transport loss. Connecting to somebody else's session stays unowned. + if (ownsSource || (options.provider === "browserbase" && options.origin === "launched")) { try { await closeSource(options.source); } catch (cleanupError) { diff --git a/packages/sdk-ts/src/browserbaseExtension.ts b/packages/sdk-ts/src/browserbaseExtension.ts index 7a6184690a..5b04476a31 100644 --- a/packages/sdk-ts/src/browserbaseExtension.ts +++ b/packages/sdk-ts/src/browserbaseExtension.ts @@ -14,7 +14,10 @@ export type ProvisionedBrowserbaseExtension = { export type BrowserbaseExtensionSdk = { extensions: { - create(params: { file: ReturnType }): Promise<{ id: string }>; + create( + params: { file: ReturnType }, + options?: { maxRetries?: number }, + ): Promise<{ id: string }>; delete( extensionId: string, options?: { headers?: Record }, @@ -31,9 +34,10 @@ export function createBrowserbaseExtensionClient( const browserbase = createSdk(apiKey); return { async uploadExtension(archivePath) { - const extension = await browserbase.extensions.create({ - file: createReadStream(archivePath), - }); + const extension = await browserbase.extensions.create( + { file: createReadStream(archivePath) }, + { maxRetries: 0 }, + ); return { id: extension.id }; }, async deleteExtension(extensionId) { @@ -44,16 +48,54 @@ export function createBrowserbaseExtensionClient( }; } +/** + * Upload attempts and the pause between them. Many Stagehand sessions + * launching at once each upload the same archive; Browserbase rejects part + * of such a burst, and one rejection used to kill the session for good. + */ +const UPLOAD_ATTEMPTS = 4; +const UPLOAD_BACKOFF_MS = [500, 1500, 4000]; + +export type ProvisionBrowserbaseExtensionOptions = { + attempts?: number; + /** Test seam; defaults to a real delay. */ + sleep?: (ms: number) => Promise; +}; + export async function provisionBrowserbaseExtension( client: BrowserbaseExtensionClient, archivePath = STAGEHAND_EXTENSION_ARCHIVE_PATH, + options: ProvisionBrowserbaseExtensionOptions = {}, ): Promise { - let uploaded: { id: string }; + const attempts = options.attempts ?? UPLOAD_ATTEMPTS; + if (!Number.isInteger(attempts) || attempts < 1 || attempts > UPLOAD_ATTEMPTS) { + throw new Error(`attempts must be an integer between 1 and ${UPLOAD_ATTEMPTS}`); + } + const sleep = options.sleep ?? ((ms: number) => new Promise((r) => setTimeout(r, ms))); + let uploaded: { id: string } | undefined; + let lastError: unknown; + let attemptsMade = 0; - try { - uploaded = await client.uploadExtension(archivePath); - } catch (error) { - throw new Error("Failed to upload the Stagehand extension to Browserbase", { cause: error }); + for (let attempt = 0; attempt < attempts && uploaded === undefined; attempt += 1) { + attemptsMade += 1; + try { + uploaded = await client.uploadExtension(archivePath); + } catch (error) { + lastError = error; + // Retry an explicit rate-limit rejection only. A transport/server error + // may follow a successful create whose ID we never received. + if (!error || typeof error !== "object" || !("status" in error) || error.status !== 429) + break; + if (attempt + 1 < attempts) { + await sleep(UPLOAD_BACKOFF_MS[Math.min(attempt, UPLOAD_BACKOFF_MS.length - 1)]!); + } + } + } + if (uploaded === undefined) { + throw new Error( + `Failed to upload the Stagehand extension to Browserbase after ${attemptsMade} attempt(s).`, + { cause: lastError }, + ); } const extensionId = uploaded.id.trim(); diff --git a/packages/sdk-ts/tests/browser/factories.test.ts b/packages/sdk-ts/tests/browser/factories.test.ts index 8511007fc4..947aae8d17 100644 --- a/packages/sdk-ts/tests/browser/factories.test.ts +++ b/packages/sdk-ts/tests/browser/factories.test.ts @@ -459,6 +459,39 @@ describe("Stagehand browser factories", () => { expect(closeSource).not.toHaveBeenCalled(); }); + it.each([ + { origin: "launched" as const, expectedSessionCloses: 1 }, + { origin: "connected" as const, expectedSessionCloses: 0 }, + ])( + "releases only a newly $origin Browserbase session when CDP attach fails", + async ({ origin, expectedSessionCloses }) => { + const closeSession = vi.fn(async () => {}); + const session = { + sessionId: "session_123", + cdpUrl: "wss://connect.browserbase.com/devtools/browser/session_123", + close: closeSession, + }; + const attachError = new Error("extension attach failed"); + const { browserbase } = createBrowserFactoriesForTest({ + createBrowserbaseSessionClient: () => ({ + createSession: async () => session, + connectSession: async () => session, + }), + connectCdp: async () => { + throw attachError; + }, + }); + + const connecting = + origin === "launched" + ? browserbase.launch({ apiKey: "bb_key", keepAlive: true }) + : browserbase.connect({ apiKey: "bb_key", sessionId: session.sessionId }); + + await expect(connecting).rejects.toBe(attachError); + expect(closeSession).toHaveBeenCalledTimes(expectedSessionCloses); + }, + ); + it("closes a connector that resolves after the internal lifecycle deadline", async () => { vi.useFakeTimers(); const connection = deferred(); diff --git a/packages/sdk-ts/tests/browserbaseExtension.test.ts b/packages/sdk-ts/tests/browserbaseExtension.test.ts index da550bdac9..daf8b4b5fa 100644 --- a/packages/sdk-ts/tests/browserbaseExtension.test.ts +++ b/packages/sdk-ts/tests/browserbaseExtension.test.ts @@ -6,6 +6,33 @@ import { } from "../src/browserbaseExtension.js"; describe("Browserbase extension client", () => { + it.each([0, -1, 1.5, 5, NaN, Infinity])( + "rejects an unbounded upload attempt count %s", + async (attempts) => { + const uploadExtension = vi.fn(); + await expect( + provisionBrowserbaseExtension( + { uploadExtension, deleteExtension: vi.fn() }, + "/archive.zip", + { attempts }, + ), + ).rejects.toThrow("attempts must be"); + expect(uploadExtension).not.toHaveBeenCalled(); + }, + ); + + it("does not print credential-bearing upload errors in the public startup message", async () => { + const secretError = new Error("https://example.test?apiKey=private-key"); + const error = await provisionBrowserbaseExtension( + { uploadExtension: vi.fn().mockRejectedValue(secretError), deleteExtension: vi.fn() }, + "/archive.zip", + { attempts: 1 }, + ).catch((error: Error) => error); + expect(error).toBeInstanceOf(Error); + expect((error as Error).message).not.toContain("private-key"); + expect((error as Error).cause).toBe(secretError); + }); + it("maps extension upload and deletion to the official SDK surface", async () => { const create = vi.fn(async () => ({ id: "ext_uploaded" })); const remove = vi.fn(async () => {}); @@ -20,7 +47,7 @@ describe("Browserbase extension client", () => { await client.deleteExtension("ext_uploaded"); expect(createSdk).toHaveBeenCalledWith("bb_key"); - expect(create).toHaveBeenCalledWith({ file: expect.anything() }); + expect(create).toHaveBeenCalledWith({ file: expect.anything() }, { maxRetries: 0 }); expect(remove).toHaveBeenCalledWith("ext_uploaded", { headers: { "Content-Type": null }, }); @@ -44,25 +71,62 @@ describe("Browserbase extension provisioning", () => { expect(deleteExtension).toHaveBeenCalledWith("ext_uploaded"); }); - it("preserves an upload failure as the cause", async () => { - const uploadError = new Error("Browserbase unavailable"); - const client: BrowserbaseExtensionClient = { - async uploadExtension() { - throw uploadError; - }, - async deleteExtension() {}, - }; + it("preserves an upload failure as the cause and names it in the message", async () => { + const uploadError = Object.assign(new Error("Rate limited"), { status: 429 }); + const uploadExtension = vi.fn(async () => { + throw uploadError; + }); + const client: BrowserbaseExtensionClient = { uploadExtension, async deleteExtension() {} }; - const error = await provisionBrowserbaseExtension(client, import.meta.filename).catch( - (caught: unknown) => caught, - ); + const error = await provisionBrowserbaseExtension(client, import.meta.filename, { + sleep: async () => {}, + }).catch((caught: unknown) => caught); expect(error).toBeInstanceOf(Error); expect((error as Error).message).toBe( - "Failed to upload the Stagehand extension to Browserbase", + "Failed to upload the Stagehand extension to Browserbase after 4 attempt(s).", ); expect((error as Error).cause).toBe(uploadError); + expect(uploadExtension).toHaveBeenCalledTimes(4); + }); + + it("retries a burst-rejected upload with backoff before giving up", async () => { + const uploadExtension = vi + .fn<() => Promise<{ id: string }>>() + .mockRejectedValueOnce(Object.assign(new Error("429 Too Many Requests"), { status: 429 })) + .mockRejectedValueOnce(Object.assign(new Error("429 Too Many Requests"), { status: 429 })) + .mockResolvedValueOnce({ id: "ext_after_retry" }); + const sleeps: number[] = []; + const client: BrowserbaseExtensionClient = { uploadExtension, async deleteExtension() {} }; + + const provisioned = await provisionBrowserbaseExtension(client, import.meta.filename, { + sleep: async (ms) => void sleeps.push(ms), + }); + expect(provisioned.extensionId).toBe("ext_after_retry"); + expect(uploadExtension).toHaveBeenCalledTimes(3); + expect(sleeps).toStrictEqual([500, 1500]); }); + it.each([401, 403, 500, 503, undefined])( + "does not replay ambiguous or permanent upload failures (%s)", + async (status) => { + const failure = Object.assign(new Error("upload failed"), { status }); + const uploadExtension = vi.fn().mockRejectedValue(failure); + const sleep = vi.fn(); + await expect( + provisionBrowserbaseExtension( + { uploadExtension, deleteExtension: vi.fn() }, + "/archive.zip", + { sleep }, + ), + ).rejects.toMatchObject({ + message: "Failed to upload the Stagehand extension to Browserbase after 1 attempt(s).", + cause: failure, + }); + expect(uploadExtension).toHaveBeenCalledOnce(); + expect(sleep).not.toHaveBeenCalled(); + }, + ); + it("rejects an empty extension ID", async () => { const client: BrowserbaseExtensionClient = { async uploadExtension() {