From 79aa86658d918eb69ed2e439342fe4936129e7ad Mon Sep 17 00:00:00 2001 From: miguel Date: Sun, 30 Aug 2026 21:16:06 -0700 Subject: [PATCH 01/49] feat(integrations): facade run executes the Playwright batch surface Sync facade/runtime.ts with the evals playwrightCompatRuntime experiment: accessibility-tree fallback for getByRole misses, shadow-root XPath resolution for snapshot-derived paths, 10 s default locator timeout. Keep the facade's browser.close() -> closeRequested semantics. run now returns a full batch envelope (telemetry, screenshot artifacts, closeRequested, batch runtime), writes page.screenshot({ path }) files, retries without a page target when the batch page vanished, and reports telemetry to stderr as stagehand_playwright_compat lines. Tool description and FACADE_AGENT_INSTRUCTIONS move to the Playwright idiom (page/context/browser in scope, no AI methods). The previous text is preserved verbatim as LEGACY_* and selectable with --surface=legacy. --- .../integrations/core/src/facade/contract.ts | 46 ++- .../integrations/core/src/facade/index.ts | 12 +- .../integrations/core/src/facade/runtime.ts | 313 +++++++++++++++++- .../core/src/facade/stdio-server.ts | 14 +- .../integrations/core/src/facade/tools.ts | 92 ++++- .../core/tests/facade-contract.test.ts | 44 +++ .../core/tests/facade-server.test.ts | 24 +- .../core/tests/facade-tools.test.ts | 180 ++++++++++ 8 files changed, 698 insertions(+), 27 deletions(-) create mode 100644 packages/integrations/core/tests/facade-tools.test.ts diff --git a/packages/integrations/core/src/facade/contract.ts b/packages/integrations/core/src/facade/contract.ts index 97abd1a13f..66ab2fdaaf 100644 --- a/packages/integrations/core/src/facade/contract.ts +++ b/packages/integrations/core/src/facade/contract.ts @@ -39,6 +39,13 @@ const actionSchema = (op: string, extra: Record> }); export const RUN_TOOL_DESCRIPTION = + 'Browse and automate websites in the persistent browser by executing JavaScript against a Playwright-shaped API. The code runs inside an async function with page, context, and browser in scope (Playwright Page, BrowserContext, and Browser); use await directly and return a JSON-serializable value when useful. Navigate with await page.goto("https://example.com"); there is no separate navigate or start tool. Alternatively, provide a batch of actions using IDs from the latest snapshot. Provide exactly one of code or actions. Each action must use "op" (never "kind") and "id" (never "ref"). Copy the bracketed snapshot ID as a string. Examples: {"actions":[{"op":"click","id":"1-42"}]}, {"actions":[{"op":"fill","id":"2-14","value":"Miami"}]}, {"actions":[{"op":"select","id":"3-9","values":"Lowest price"}]}.'; + +/** + * Run-tool description of the pre-Playwright-idiom facade surface, kept + * byte-identical so `--surface=legacy` hosts reproduce the earlier contract. + */ +export const LEGACY_RUN_TOOL_DESCRIPTION = 'Browse and automate websites in the persistent Stagehand browser. Navigate with JavaScript such as await page.goto("https://example.com"); there is no separate navigate or start tool. Execute either a JavaScript workflow against the Stagehand Playwright facade or a batch of actions using IDs from the latest snapshot. Provide exactly one of code or actions. Each action must use "op" (never "kind") and "id" (never "ref"). Copy the bracketed snapshot ID as a string. Examples: {"actions":[{"op":"click","id":"1-42"}]}, {"actions":[{"op":"fill","id":"2-14","value":"Miami"}]}, {"actions":[{"op":"select","id":"3-9","values":"Lowest price"}]}.'; export const SNAPSHOT_TOOL_DESCRIPTION = @@ -117,12 +124,49 @@ export const FACADE_TOOLS = [ }, ] as const; +export const FACADE_LEGACY_TOOLS = [ + { name: "run", description: LEGACY_RUN_TOOL_DESCRIPTION, inputSchema: RUN_INPUT_SCHEMA }, + FACADE_TOOLS[1], + FACADE_TOOLS[2], +] as const; + +export type FacadeSurface = "playwright" | "legacy"; + +const SURFACE_FLAG = "--surface="; + +/** `--surface=legacy` selects the pre-Playwright-idiom run description; default is "playwright". */ +export function facadeSurfaceFromArgs(args: string[]): FacadeSurface { + const value = args.find((arg) => arg.startsWith(SURFACE_FLAG)); + if (value === undefined) return "playwright"; + const surface = value.slice(SURFACE_FLAG.length); + if (surface !== "playwright" && surface !== "legacy") { + throw new Error(`${SURFACE_FLAG} must be "playwright" or "legacy".`); + } + return surface; +} + +export function facadeToolsForSurface(surface: FacadeSurface) { + return surface === "legacy" ? FACADE_LEGACY_TOOLS : FACADE_TOOLS; +} + /** * Canonical agent system prompt for the facade tool surface. Host examples * (Eve, Vercel AI SDK, deepagents) should use this text rather than authoring * their own so agent guidance stays identical across frameworks. */ -export const FACADE_AGENT_INSTRUCTIONS = `You control one persistent browser through exactly three tools: +export const FACADE_AGENT_INSTRUCTIONS = `Browser tool surface: Stagehand Playwright facade. +You control one persistent browser through exactly three tools: +- run: execute JavaScript against an initialized Playwright page, context, and browser (page.goto, page.locator(selector).click()/fill(), page.getByRole(...), page.evaluate(...), page.waitForURL(...), and the rest of the Playwright API). Use await directly and return JSON-serializable values so you can inspect progress. Alternatively, pass snapshot actions. +- snapshot: inspect the active page's accessibility tree and hydrate bracketed element IDs for run actions. +- screenshot: inspect the rendered page visually. + +Pass run exactly one of code or actions; every action uses "op" and "id", never "kind" or "ref". Snapshot IDs are valid only for the latest snapshot of the active page; snapshot again after navigation or stale IDs. The first browser action should usually be: await page.goto(url, { waitUntil: 'domcontentloaded' }). Do not launch another browser or create a separate browser process.`; + +/** + * Agent instructions of the pre-Playwright-idiom facade surface, kept + * byte-identical for `--surface=legacy` hosts. + */ +export const LEGACY_FACADE_AGENT_INSTRUCTIONS = `You control one persistent browser through exactly three tools: - snapshot: inspect the active page and hydrate bracketed element IDs. - run: provide either snapshot actions or JavaScript using the Playwright-shaped page API. - screenshot: inspect the rendered page visually. diff --git a/packages/integrations/core/src/facade/index.ts b/packages/integrations/core/src/facade/index.ts index 6bb6f38877..26b49699bf 100644 --- a/packages/integrations/core/src/facade/index.ts +++ b/packages/integrations/core/src/facade/index.ts @@ -1,7 +1,13 @@ export { FACADE_TOOLS, + FACADE_LEGACY_TOOLS, FACADE_AGENT_INSTRUCTIONS, + LEGACY_FACADE_AGENT_INSTRUCTIONS, RUN_TOOL_DESCRIPTION, + LEGACY_RUN_TOOL_DESCRIPTION, + facadeSurfaceFromArgs, + facadeToolsForSurface, + type FacadeSurface, SNAPSHOT_TOOL_DESCRIPTION, SCREENSHOT_TOOL_DESCRIPTION, RUN_INPUT_SCHEMA, @@ -18,7 +24,11 @@ export { type RefAction, type CodeModeRunInput, } from "./contract.js"; -export { StagehandFacadeTools } from "./tools.js"; +export { + StagehandFacadeTools, + type StagehandFacadeRunReport, + type StagehandFacadeToolsOptions, +} from "./tools.js"; export { StagehandFacadeConfigError, stagehandFacadeConfigFromEnv, diff --git a/packages/integrations/core/src/facade/runtime.ts b/packages/integrations/core/src/facade/runtime.ts index 03242d1cab..50bcf8dbb4 100644 --- a/packages/integrations/core/src/facade/runtime.ts +++ b/packages/integrations/core/src/facade/runtime.ts @@ -27,7 +27,24 @@ type QueryStep = hasNot?: QueryStep[]; visible?: boolean; } - | { kind: "nth"; index: number }; + | { kind: "nth"; index: number } + /** + * A `role` step that was resolved against the browser's accessibility tree. + * `values` are document-relative XPaths for the nodes whose role and name + * matched; the state filters are carried over from the original role step. + */ + | { + kind: "xpaths"; + values: string[]; + checked?: boolean; + disabled?: boolean; + selected?: boolean; + expanded?: boolean; + pressed?: boolean; + level?: number; + }; + +type RoleStep = Extract; type RawLocator = { click(options?: { button?: "left" | "right" | "middle"; clickCount?: number }): Promise; @@ -142,6 +159,137 @@ export async function createPlaywrightCompatRuntime( ? { kind: "regexp", source: value.source, flags: value.flags } : { kind: "string", value: String(value), exact }; + // --------------------------------------------------------------------------- + // getByRole fallback through the accessibility tree. + // + // The in-page role matcher reimplements accessible-name computation and + // disagrees with Chrome's on real sites (descendant aria-label / alt / svg + // titles, labelledby across shadow roots, custom elements). When a plan that + // contains a role step matches nothing in the DOM, resolve the role step + // against `page.snapshot()` — the same accessibility tree the `snapshot` + // tool shows the agent — and re-run the plan with those nodes' XPaths. + // --------------------------------------------------------------------------- + + /** Playwright role → roles as they appear in Stagehand's formatted tree. */ + const ACCESSIBILITY_ROLE_ALIASES: Record = { + img: ["image", "img"], + image: ["image", "img"], + textbox: ["textbox", "searchbox"], + cell: ["cell", "gridcell"], + gridcell: ["gridcell", "cell"], + }; + + const ACCESSIBILITY_FALLBACK_CACHE_TTL_MS = 750; + + type AccessibilityTreeNode = { id: string; role: string; name: string }; + + const parseAccessibilityTree = (formattedTree: string): AccessibilityTreeNode[] => { + const nodes: AccessibilityTreeNode[] = []; + for (const rawLine of formattedTree.split("\n")) { + const line = rawLine.match(/^\s*\[([^\]]+)\]\s+(.*)$/u); + if (!line) continue; + let rest = line[2] ?? ""; + // Trailing state flags rendered by formatStateFlags. + rest = rest.replace(/(?:\s\[(?:selected|checked)\])+$/u, ""); + const separator = rest.indexOf(": "); + const roleToken = separator === -1 ? rest : rest.slice(0, separator); + const name = separator === -1 ? "" : rest.slice(separator + 2); + // "scrollable, html" style lines carry the role before the comma. + const role = roleToken.split(",")[0]?.trim() ?? ""; + if (!role) continue; + nodes.push({ id: line[1] ?? "", role, name }); + } + return nodes; + }; + + const matchesAccessibleName = (value: string, expected: JsonMatcher): boolean => { + const normalized = value.replace(/\s+/gu, " ").trim(); + if (expected.kind === "regexp") { + return new RegExp(expected.source, expected.flags).test(normalized); + } + const target = expected.value.replace(/\s+/gu, " ").trim(); + return expected.exact + ? normalized === target + : normalized.toLocaleLowerCase().includes(target.toLocaleLowerCase()); + }; + + const planHasRoleStep = (plan: QueryStep[]): boolean => + plan.some( + (step) => + step.kind === "role" || + (step.kind === "filter" && + ((step.has && planHasRoleStep(step.has)) || + (step.hasNot && planHasRoleStep(step.hasNot)))), + ); + + const resolveRoleStepWithTree = ( + step: RoleStep, + nodes: AccessibilityTreeNode[], + xpathMap: Record, + ): QueryStep | null => { + const roles = new Set(ACCESSIBILITY_ROLE_ALIASES[step.role] ?? [step.role]); + const values = nodes + .filter( + (node) => + roles.has(node.role) && (!step.name || matchesAccessibleName(node.name, step.name)), + ) + .map((node) => xpathMap[node.id]) + .filter((xpath): xpath is string => typeof xpath === "string" && xpath.length > 0); + if (values.length === 0) return null; + const { kind: _kind, role: _role, name: _name, includeHidden: _hidden, ...state } = step; + return { kind: "xpaths", values, ...state }; + }; + + /** + * Returns a copy of `plan` whose top-level role steps are replaced by + * accessibility-tree resolved XPath steps, or null when the tree has no + * candidate for at least one of them (or no tree is available). + */ + const resolvePlanWithAccessibilityTree = async ( + page: RawPage, + plan: QueryStep[], + ): Promise => { + if (typeof page.snapshot !== "function") { + record("misses", "getByRole.accessibilityTree:snapshotUnavailable"); + return null; + } + let snapshot: unknown; + try { + snapshot = await page.snapshot({ includeIframes: false }); + } catch { + record("misses", "getByRole.accessibilityTree:snapshotError"); + return null; + } + const tree = snapshot as { formattedTree?: unknown; xpathMap?: unknown } | null; + if ( + !tree || + typeof tree.formattedTree !== "string" || + !tree.xpathMap || + typeof tree.xpathMap !== "object" + ) { + record("misses", "getByRole.accessibilityTree:noTree"); + return null; + } + const nodes = parseAccessibilityTree(tree.formattedTree); + const xpathMap = tree.xpathMap as Record; + let replaced = false; + const resolved: QueryStep[] = []; + for (const step of plan) { + if (step.kind !== "role") { + resolved.push(step); + continue; + } + const fallback = resolveRoleStepWithTree(step, nodes, xpathMap); + if (!fallback) { + record("misses", "getByRole.accessibilityTree:noCandidates"); + return null; + } + resolved.push(fallback); + replaced = true; + } + return replaced ? resolved : null; + }; + const unsupported = (surface: string, method: PropertyKey): never => { const name = `${surface}.${String(method)}`; record("misses", name); @@ -251,6 +399,72 @@ export async function createPlaywrightCompatRuntime( } return elements; }; + /** + * Resolve an XPath produced by Stagehand's accessibility snapshot. Those + * paths are positional (`/html[1]/body[1]/x-host[1]//div[2]/button[1]`) + * and encode a shadow-root boundary as `//`, which native + * `document.evaluate` cannot follow. Mirrors the extension's + * resolveStagehandShadowHopMatches: child steps walk light-DOM children, + * a `//` step after the first walks into the host's (open) shadow root. + */ + const resolveStagehandXPath = (expression: string): Element[] => { + const path = expression.trim().replace(/^xpath=/iu, ""); + if (!path) return []; + type Step = { hop: boolean; tag: string; index?: number }; + const steps: Step[] = []; + let cursor = 0; + while (cursor < path.length) { + let hop = false; + if (path.startsWith("//", cursor)) { + hop = true; + cursor += 2; + } else if (path[cursor] === "/") { + cursor += 1; + } + const start = cursor; + while (cursor < path.length && path[cursor] !== "/") cursor += 1; + const raw = path.slice(start, cursor).trim(); + if (!raw) continue; + const parsed = raw.match(/^([^[]+)(?:\[(\d+)\])?$/u); + if (!parsed) return queryXPath(document, path); + steps.push({ + hop, + tag: (parsed[1] ?? "*").toLowerCase(), + ...(parsed[2] ? { index: Number(parsed[2]) } : {}), + }); + } + const hasShadowHop = steps.some((step, position) => step.hop && position > 0); + if (!hasShadowHop) { + try { + return queryXPath(document, path); + } catch { + return []; + } + } + let current: Array = [document]; + for (const [position, step] of steps.entries()) { + const next: Element[] = []; + for (const root of current) { + let pool: Element[]; + if (root instanceof Document) { + pool = root.documentElement ? [root.documentElement] : []; + } else if (step.hop && position > 0) { + pool = root instanceof Element ? [...(root.shadowRoot?.children ?? [])] : []; + } else { + pool = [...root.children]; + } + const tagged = pool.filter( + (element) => step.tag === "*" || element.localName.toLowerCase() === step.tag, + ); + const picked = + step.index === undefined ? tagged : [tagged[step.index - 1]!].filter(Boolean); + for (const element of picked) if (!next.includes(element)) next.push(element); + } + if (!next.length) return []; + current = next; + } + return current as Element[]; + }; const splitSelectorList = (selector: string): string[] => { const parts: string[] = []; let start = 0; @@ -444,6 +658,50 @@ export async function createPlaywrightCompatRuntime( return false; return true; }); + } else if (step.kind === "xpaths") { + // Candidates come from the accessibility tree, so role, name, and + // visibility are already settled; only scope and state remain. + const scoped = dedupe( + step.values.flatMap((expression) => { + try { + return resolveStagehandXPath(expression); + } catch { + return []; + } + }), + ).filter((element) => + roots.some((root) => root instanceof Document || root.contains(element)), + ); + current = scoped.filter((element) => { + if ( + step.checked !== undefined && + (element as HTMLInputElement).checked !== step.checked + ) + return false; + if ( + step.disabled !== undefined && + (element as HTMLInputElement).disabled !== step.disabled + ) + return false; + if ( + step.selected !== undefined && + (element as HTMLOptionElement).selected !== step.selected + ) + return false; + if ( + step.expanded !== undefined && + element.getAttribute("aria-expanded") !== String(step.expanded) + ) + return false; + if ( + step.pressed !== undefined && + element.getAttribute("aria-pressed") !== String(step.pressed) + ) + return false; + if (step.level !== undefined && Number(element.tagName.slice(1)) !== step.level) + return false; + return true; + }); } else if (step.kind === "filter") { current = current.filter((element) => { const text = element.textContent ?? ""; @@ -939,7 +1197,12 @@ export async function createPlaywrightCompatRuntime( options: { timeout?: number } = {}, ): Promise { record("calls", method); - const timeout = options.timeout ?? 30_000; + // Playwright's own default is 30 s, but through this facade a miss is a + // dead wait with no call log: eval traces showed 0.65 such misses per + // task at 30 s each, with agents treating the silence as "not on the + // page". 10 s (the original facade default) bounds that tail; callers + // that genuinely need longer pass `timeout` explicitly. + const timeout = options.timeout ?? 10_000; const deadline = Date.now() + timeout; let result: QueryResult = { count: 0 }; let lastActionError: unknown; @@ -1212,22 +1475,45 @@ export async function createPlaywrightCompatRuntime( const key = pageKey(page); const existing = compatPages.get(key); if (existing) return existing; + const accessibilityFallbacks = new Map(); const state: PageState = { rawPage: page, cachedUrl: await page.url(), viewport: await page.evaluate("({ width: innerWidth, height: innerHeight })"), closed: false, execute: async (plan, operation, extra = {}) => { - const result = await page.evaluate( - buildQueryEvaluationExpression({ plan, operation, ...extra }), - ); - if (result.error) { - const error = new Error(result.error.message); - error.name = result.error.name; - if (result.error.stack) error.stack = result.error.stack; - throw error; + const run = async (steps: QueryStep[]): Promise => { + const result = await page.evaluate( + buildQueryEvaluationExpression({ plan: steps, operation, ...extra }), + ); + if (result.error) { + const error = new Error(result.error.message); + error.name = result.error.name; + if (result.error.stack) error.stack = result.error.stack; + throw error; + } + return result; + }; + const result = await run(plan); + if (result.count !== 0 || plan.length === 0 || !planHasRoleStep(plan)) return result; + + // Nothing matched in the DOM; consult the accessibility tree. Cache per + // plan briefly so the inspect → tag → action sequence inside one + // locator action (and the 50 ms retry loop) reuses a single snapshot. + const cacheKey = JSON.stringify(plan); + const cached = accessibilityFallbacks.get(cacheKey); + let resolved: QueryStep[] | null; + if (cached && Date.now() - cached.at < ACCESSIBILITY_FALLBACK_CACHE_TTL_MS) { + resolved = cached.plan; + } else { + resolved = await resolvePlanWithAccessibilityTree(page, plan); + accessibilityFallbacks.set(cacheKey, { at: Date.now(), plan: resolved }); } - return result; + if (!resolved) return result; + const fallbackResult = await run(resolved); + if (fallbackResult.count > 0) record("calls", "locator.getByRole.accessibilityTree"); + else record("misses", "getByRole.accessibilityTree:unresolvedXPath"); + return fallbackResult; }, refreshUrl: async () => { state.cachedUrl = await page.url(); @@ -2006,8 +2292,9 @@ export async function createPlaywrightCompatRuntime( contexts: () => [context], isConnected: () => !closeRequested, // Do not close Chrome from inside experimentalBatch — the callback is - // running in that browser. Host-side run() reads closeRequested() and - // tears down Stagehand + the keep-alive session after the batch returns. + // running in that browser. The host reads closeRequested() from the batch + // envelope and tears down Stagehand + the keep-alive session after the + // batch returns. close: async () => { record("calls", "browser.close"); closeRequested = true; diff --git a/packages/integrations/core/src/facade/stdio-server.ts b/packages/integrations/core/src/facade/stdio-server.ts index 53b1bc5bac..84884e7e9e 100644 --- a/packages/integrations/core/src/facade/stdio-server.ts +++ b/packages/integrations/core/src/facade/stdio-server.ts @@ -14,7 +14,8 @@ import { sanitizeErrorMessage } from "../harness/redact.js"; import { stagehandFacadeConfigFromEnv } from "./config.js"; import { CodeModeRunInputSchema, - FACADE_TOOLS, + facadeSurfaceFromArgs, + facadeToolsForSurface, SCREENSHOT_TOOL_DESCRIPTION, SNAPSHOT_TOOL_DESCRIPTION, ScreenshotInputSchema, @@ -34,12 +35,13 @@ type FacadeResources = { const server = new McpServer({ name: "stagehand-facade", version: "4.0.0" }); const screenshotBase64Budget = screenshotBase64BudgetFromArgs(process.argv.slice(2)); +const facadeTools = facadeToolsForSurface(facadeSurfaceFromArgs(process.argv.slice(2))); let resourcesPromise: Promise | undefined; let closing = false; server.registerTool( "run", - { description: FACADE_TOOLS[0].description, inputSchema: CodeModeRunInputSchema }, + { description: facadeTools[0].description, inputSchema: CodeModeRunInputSchema }, async () => ({ content: [] }), ); server.registerTool( @@ -54,7 +56,7 @@ server.registerTool( ); server.server.removeRequestHandler("tools/list"); -server.server.setRequestHandler(ListToolsRequestSchema, () => ({ tools: [...FACADE_TOOLS] })); +server.server.setRequestHandler(ListToolsRequestSchema, () => ({ tools: [...facadeTools] })); server.server.removeRequestHandler("tools/call"); server.server.setRequestHandler(CallToolRequestSchema, async (request) => { try { @@ -125,7 +127,11 @@ async function createResources(): Promise { : await localBrowser.launch(config.browser.launchOptions); try { const stagehand = await Stagehand.create({ browser, ...config.stagehand }); - return { browser, stagehand, tools: new StagehandFacadeTools(stagehand) }; + const tools = new StagehandFacadeTools(stagehand, { + onRunReport: (report) => + process.stderr.write(`stagehand_playwright_compat ${JSON.stringify(report)}\n`), + }); + return { browser, stagehand, tools }; } catch (error) { await browser.close().catch(() => undefined); throw error; diff --git a/packages/integrations/core/src/facade/tools.ts b/packages/integrations/core/src/facade/tools.ts index e291407d3f..db7b7edfef 100644 --- a/packages/integrations/core/src/facade/tools.ts +++ b/packages/integrations/core/src/facade/tools.ts @@ -1,3 +1,5 @@ +import fsp from "node:fs/promises"; +import path from "node:path"; import type { ExperimentalBatchCallback, Page, Stagehand } from "@browserbasehq/stagehand"; import { NAVIGATED_SNAPSHOT_ERROR, @@ -6,17 +8,40 @@ import { staleSnapshotIdError, type RefAction, } from "./contract.js"; -import { createPlaywrightCompatRuntime } from "./runtime.js"; +import { createPlaywrightCompatRuntime, type PlaywrightCompatTelemetry } from "./runtime.js"; type SnapshotState = { url: string; xpathById: Record }; type HydratedAction = RefAction & { selector: string }; type ActionResult = { completed: number }; +type ScreenshotArtifact = { path: string; base64: string }; type RunEnvelope = { __stagehandPlaywrightCompat: true; value: unknown; executionError?: { name: string; message: string; stack?: string }; + telemetry: PlaywrightCompatTelemetry; + artifacts: ScreenshotArtifact[]; + closeRequested: boolean; + batchRuntimeMs: number; }; +export type StagehandFacadeRunReport = { + telemetry: PlaywrightCompatTelemetry; + /** Wall-clock time of the whole experimentalBatch round trip. */ + batchRoundTripMs: number; + /** Time the agent's code spent executing inside the batch. */ + batchRuntimeMs: number; + closeRequested: boolean; +}; + +export type StagehandFacadeToolsOptions = { + /** Directory that relative `page.screenshot({ path })` paths resolve against. Defaults to process.cwd(). */ + artifactRoot?: string; + /** Observes every completed `run` batch (including ones whose code threw). */ + onRunReport?: (report: StagehandFacadeRunReport) => void; +}; + +const RUN_BATCH_TIMEOUT_MS = 60_000; + export type StagehandFacadeScreenshot = { data: string; mimeType: "image/png" | "image/jpeg"; @@ -60,6 +85,7 @@ const page = runtime.page; const context = runtime.context; const browser = runtime.browser; const console = globalThis.console; +const __stagehandBatchStartedAt = performance.now(); let value; let executionError; try { @@ -78,13 +104,20 @@ return { __stagehandPlaywrightCompat: true, value, executionError, + telemetry: runtime.telemetry(), + artifacts: runtime.artifacts(), + closeRequested: runtime.closeRequested(), + batchRuntimeMs: performance.now() - __stagehandBatchStartedAt, };`; export class StagehandFacadeTools { private readonly snapshotsByPage = new Map(); private queue: Promise = Promise.resolve(); - constructor(private readonly stagehand: Stagehand) {} + constructor( + private readonly stagehand: Stagehand, + private readonly options: StagehandFacadeToolsOptions = {}, + ) {} snapshot(options: { includeIframes?: boolean } = {}): Promise { return this.enqueue(() => this.snapshotNow(options)); @@ -166,11 +199,15 @@ export class StagehandFacadeTools { "input", FACADE_PRELUDE + code + FACADE_EPILOGUE, ) as ExperimentalBatchCallback, RunEnvelope>; - const envelope = await this.stagehand.experimentalBatch( - callback, - {}, - { page, timeout: 60_000 }, - ); + const startedAt = performance.now(); + const envelope = await this.runBatchWithActivePageFallback(callback, page); + this.options.onRunReport?.({ + telemetry: envelope.telemetry, + batchRoundTripMs: performance.now() - startedAt, + batchRuntimeMs: envelope.batchRuntimeMs, + closeRequested: envelope.closeRequested, + }); + await this.writeScreenshotArtifacts(envelope.artifacts); if (envelope.executionError) { const error = new Error(envelope.executionError.message); error.name = envelope.executionError.name; @@ -180,6 +217,47 @@ export class StagehandFacadeTools { return envelope.value; } + /** + * The batch controller resolves its target page before invoking the + * callback, so when the active page vanished between activePage() and the + * batch (tab closed by the previous snippet) this retry cannot replay + * partially executed agent code. + */ + private async runBatchWithActivePageFallback( + callback: ExperimentalBatchCallback, RunEnvelope>, + page: Page, + ): Promise { + try { + return await this.stagehand.experimentalBatch( + callback, + {}, + { page, timeout: RUN_BATCH_TIMEOUT_MS }, + ); + } catch (error) { + if (!(error instanceof Error) || !/callback batch page was not found/iu.test(error.message)) { + throw error; + } + const context = this.stagehand.browser.context; + if (!(await context.activePage())) await context.newPage(); + return await this.stagehand.experimentalBatch( + callback, + {}, + { timeout: RUN_BATCH_TIMEOUT_MS }, + ); + } + } + + private async writeScreenshotArtifacts(artifacts: ScreenshotArtifact[]): Promise { + const root = this.options.artifactRoot ?? process.cwd(); + for (const artifact of artifacts) { + const target = path.isAbsolute(artifact.path) + ? artifact.path + : path.resolve(root, artifact.path); + await fsp.mkdir(path.dirname(target), { recursive: true }); + await fsp.writeFile(target, Buffer.from(artifact.base64, "base64")); + } + } + private async activePage(): Promise { const page = await this.stagehand.browser.context.activePage(); if (!page) throw new Error("Stagehand has no active page."); diff --git a/packages/integrations/core/tests/facade-contract.test.ts b/packages/integrations/core/tests/facade-contract.test.ts index 05db01c772..ea9590a0af 100644 --- a/packages/integrations/core/tests/facade-contract.test.ts +++ b/packages/integrations/core/tests/facade-contract.test.ts @@ -1,7 +1,12 @@ import { describe, expect, it } from "vitest"; import { CodeModeRunInputSchema, + FACADE_AGENT_INSTRUCTIONS, + FACADE_LEGACY_TOOLS, FACADE_TOOLS, + facadeSurfaceFromArgs, + facadeToolsForSurface, + LEGACY_FACADE_AGENT_INSTRUCTIONS, NAVIGATED_SNAPSHOT_ERROR, NO_HYDRATED_SNAPSHOT_ERROR, STALE_SNAPSHOT_ID_ERROR, @@ -33,6 +38,45 @@ describe("Stagehand facade contract", () => { expect(FACADE_TOOLS[2].description).toContain('{"type":"jpeg","quality":40,"fullPage":false}'); }); + it("advertises the Playwright idiom and no Stagehand AI methods", () => { + expect(FACADE_TOOLS[0].description).toContain("page, context, and browser in scope"); + expect(FACADE_TOOLS[0].description).toContain("Playwright-shaped API"); + expect(FACADE_AGENT_INSTRUCTIONS).toContain("Playwright page, context, and browser"); + expect(FACADE_AGENT_INSTRUCTIONS).toContain("page.getByRole("); + for (const text of [FACADE_TOOLS[0].description, FACADE_AGENT_INSTRUCTIONS]) { + expect(text).not.toMatch(/\b(act|extract|observe)\(/u); + expect(text).not.toContain("stagehand."); + } + }); + + it("keeps the legacy surface byte-identical except for the run description", () => { + expect(FACADE_LEGACY_TOOLS.map((tool) => tool.name)).toStrictEqual([ + "run", + "snapshot", + "screenshot", + ]); + expect(FACADE_LEGACY_TOOLS[0].description).toBe( + 'Browse and automate websites in the persistent Stagehand browser. Navigate with JavaScript such as await page.goto("https://example.com"); there is no separate navigate or start tool. Execute either a JavaScript workflow against the Stagehand Playwright facade or a batch of actions using IDs from the latest snapshot. Provide exactly one of code or actions. Each action must use "op" (never "kind") and "id" (never "ref"). Copy the bracketed snapshot ID as a string. Examples: {"actions":[{"op":"click","id":"1-42"}]}, {"actions":[{"op":"fill","id":"2-14","value":"Miami"}]}, {"actions":[{"op":"select","id":"3-9","values":"Lowest price"}]}.', + ); + expect(FACADE_LEGACY_TOOLS[0].inputSchema).toBe(FACADE_TOOLS[0].inputSchema); + expect(FACADE_LEGACY_TOOLS[1]).toBe(FACADE_TOOLS[1]); + expect(FACADE_LEGACY_TOOLS[2]).toBe(FACADE_TOOLS[2]); + expect(LEGACY_FACADE_AGENT_INSTRUCTIONS).toContain( + "Use snapshot actions for simple interactions", + ); + expect(LEGACY_FACADE_AGENT_INSTRUCTIONS).not.toBe(FACADE_AGENT_INSTRUCTIONS); + }); + + it("selects the surface from --surface", () => { + expect(facadeSurfaceFromArgs([])).toBe("playwright"); + expect(facadeSurfaceFromArgs(["--max-screenshot-base64-bytes=4096"])).toBe("playwright"); + expect(facadeSurfaceFromArgs(["--surface=legacy"])).toBe("legacy"); + expect(facadeSurfaceFromArgs(["--surface=playwright"])).toBe("playwright"); + expect(() => facadeSurfaceFromArgs(["--surface=codemode"])).toThrow("--surface="); + expect(facadeToolsForSurface("legacy")).toBe(FACADE_LEGACY_TOOLS); + expect(facadeToolsForSurface("playwright")).toBe(FACADE_TOOLS); + }); + it("pins snapshot error punctuation", () => { expect(NO_HYDRATED_SNAPSHOT_ERROR).toBe( "No hydrated snapshot exists for the active page; call snapshot first.", diff --git a/packages/integrations/core/tests/facade-server.test.ts b/packages/integrations/core/tests/facade-server.test.ts index e7a83613d6..009f658ae3 100644 --- a/packages/integrations/core/tests/facade-server.test.ts +++ b/packages/integrations/core/tests/facade-server.test.ts @@ -3,7 +3,7 @@ import type { Stream } from "node:stream"; import { Client } from "@modelcontextprotocol/sdk/client/index.js"; import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js"; import { afterEach, beforeEach, describe, expect, it } from "vitest"; -import { FACADE_TOOLS } from "../src/facade/contract.js"; +import { FACADE_LEGACY_TOOLS, FACADE_TOOLS } from "../src/facade/contract.js"; const entrypoint = fileURLToPath(new URL("../dist/facade/stdio-server.mjs", import.meta.url)); const readyMessage = "Stagehand facade MCP host listening on stdio"; @@ -66,6 +66,28 @@ describe("built Stagehand facade stdio server", () => { }); }); +describe("built Stagehand facade stdio server with --surface=legacy", () => { + it("lists the legacy run description", async () => { + const transport = new StdioClientTransport({ + command: process.execPath, + args: [entrypoint, "--surface=legacy"], + env: { PATH: process.env.PATH ?? "", STAGEHAND_BROWSER: "invalid" }, + stderr: "pipe", + }); + if (!transport.stderr) throw new Error("stdio transport did not expose stderr"); + const ready = waitForOutput(transport.stderr, readyMessage); + const client = new Client({ name: "stagehand-facade-test", version: "1.0.0" }); + await Promise.all([client.connect(transport), ready]); + try { + const result = await client.listTools(); + expect(result.tools).toStrictEqual([...FACADE_LEGACY_TOOLS]); + expect(result.tools[0].description).not.toBe(FACADE_TOOLS[0].description); + } finally { + await client.close(); + } + }); +}); + function textContent(result: Awaited>): string { const block = result.content[0]; return block && block.type === "text" ? block.text : ""; diff --git a/packages/integrations/core/tests/facade-tools.test.ts b/packages/integrations/core/tests/facade-tools.test.ts new file mode 100644 index 0000000000..01e0e59703 --- /dev/null +++ b/packages/integrations/core/tests/facade-tools.test.ts @@ -0,0 +1,180 @@ +import fsp from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import type { Stagehand } from "@browserbasehq/stagehand"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { StagehandFacadeTools, type StagehandFacadeRunReport } from "../src/facade/tools.js"; + +type FakePage = ReturnType; + +function createFakePage(initialUrl = "about:blank") { + let currentUrl = initialUrl; + return { + pageId: "page-1", + goto: vi.fn(async (url: string) => { + currentUrl = url; + return null; + }), + url: vi.fn(async () => currentUrl), + title: vi.fn(async () => "Example Domain"), + evaluate: vi.fn(async (expression: unknown) => { + if (typeof expression === "string" && expression.includes("innerWidth")) { + return { width: 1280, height: 720 }; + } + return undefined; + }), + screenshot: vi.fn(async () => Uint8Array.from([0x89, 0x50, 0x4e, 0x47])), + close: vi.fn(async () => undefined), + on: vi.fn(async () => ({ unsubscribe: async () => undefined })), + onCDP: vi.fn(async () => ({ unsubscribe: async () => undefined })), + sendCDP: vi.fn(async () => ({})), + snapshot: vi.fn(async () => ({ formattedTree: "", xpathMap: {} })), + }; +} + +/** + * Stands in for Stagehand: experimentalBatch() invokes the callback in-process + * with a batch-shaped { page, context } so the Playwright compat runtime runs + * exactly as it would inside the extension, minus the browser. + */ +function createFakeStagehand(page: FakePage) { + const context = { + activePage: vi.fn(async (): Promise => page), + newPage: vi.fn(async () => page), + pages: vi.fn(async () => [page]), + setActivePage: vi.fn(async () => undefined), + }; + const experimentalBatch = vi.fn( + async ( + callback: (stagehand: unknown, input: unknown) => Promise, + input: unknown, + _options: { page?: unknown; timeout: number }, + ) => callback({ page, context }, input), + ); + return { + stagehand: { browser: { context }, experimentalBatch } as unknown as Stagehand, + context, + experimentalBatch, + }; +} + +describe("StagehandFacadeTools.run (Playwright batch surface)", () => { + const tempDirs: string[] = []; + + afterEach(async () => { + await Promise.all(tempDirs.map((dir) => fsp.rm(dir, { recursive: true, force: true }))); + tempDirs.length = 0; + }); + + it("executes Playwright-style code with page, context, and browser in scope", async () => { + const page = createFakePage(); + const { stagehand, experimentalBatch } = createFakeStagehand(page); + const reports: StagehandFacadeRunReport[] = []; + const tools = new StagehandFacadeTools(stagehand, { onRunReport: (r) => reports.push(r) }); + + const result = await tools.run(` + await page.goto("https://example.com", { waitUntil: "domcontentloaded" }); + return { + url: page.url(), + title: await page.title(), + contexts: browser.contexts().length, + sameContext: browser.contexts()[0] === context, + connected: browser.isConnected(), + }; + `); + + expect(result).toStrictEqual({ + url: "https://example.com", + title: "Example Domain", + contexts: 1, + sameContext: true, + connected: true, + }); + expect(page.goto).toHaveBeenCalledWith("https://example.com", { + waitUntil: "domcontentloaded", + }); + expect(experimentalBatch).toHaveBeenCalledTimes(1); + expect(experimentalBatch.mock.calls[0][2]).toStrictEqual({ page, timeout: 60_000 }); + expect(reports).toHaveLength(1); + expect(reports[0].telemetry.calls["page.goto"]).toBe(1); + expect(reports[0].closeRequested).toBe(false); + expect(reports[0].batchRuntimeMs).toBeGreaterThanOrEqual(0); + expect(reports[0].batchRoundTripMs).toBeGreaterThanOrEqual(reports[0].batchRuntimeMs); + }); + + it("does not expose Stagehand AI methods or the raw client to the snippet", async () => { + const { stagehand } = createFakeStagehand(createFakePage()); + const tools = new StagehandFacadeTools(stagehand); + + // Unknown members of the guarded page are throwing stubs, so a snippet + // that reaches for Stagehand's AI methods fails loudly instead of silently. + await expect(tools.run(`await page.act("click the button");`)).rejects.toThrow(/act/u); + await expect(tools.run(`return typeof stagehand;`)).resolves.toBe("undefined"); + }); + + it("rethrows snippet errors with their name and message", async () => { + const { stagehand } = createFakeStagehand(createFakePage()); + const reports: StagehandFacadeRunReport[] = []; + const tools = new StagehandFacadeTools(stagehand, { onRunReport: (r) => reports.push(r) }); + + await expect( + tools.run(`const error = new TypeError("boom"); throw error;`), + ).rejects.toMatchObject({ name: "TypeError", message: "boom" }); + // The report still fires so hosts can see telemetry for failed batches. + expect(reports).toHaveLength(1); + }); + + it("writes page.screenshot({ path }) artifacts under artifactRoot", async () => { + const page = createFakePage(); + const { stagehand } = createFakeStagehand(page); + const artifactRoot = await fsp.mkdtemp(path.join(os.tmpdir(), "facade-tools-")); + tempDirs.push(artifactRoot); + const tools = new StagehandFacadeTools(stagehand, { artifactRoot }); + + await tools.run(`await page.screenshot({ path: "shots/first.png" }); return "ok";`); + + const written = await fsp.readFile(path.join(artifactRoot, "shots", "first.png")); + expect([...written]).toStrictEqual([0x89, 0x50, 0x4e, 0x47]); + expect(page.screenshot).toHaveBeenCalledTimes(1); + }); + + it("reports browser.close() without closing the page inside the batch", async () => { + const page = createFakePage(); + const { stagehand } = createFakeStagehand(page); + const reports: StagehandFacadeRunReport[] = []; + const tools = new StagehandFacadeTools(stagehand, { onRunReport: (r) => reports.push(r) }); + + await expect(tools.run(`await browser.close(); return browser.isConnected();`)).resolves.toBe( + false, + ); + expect(page.close).not.toHaveBeenCalled(); + expect(reports[0].closeRequested).toBe(true); + expect(reports[0].telemetry.calls["browser.close"]).toBe(1); + }); + + it("retries without a page target when the batch page vanished", async () => { + const page = createFakePage(); + const { stagehand, context, experimentalBatch } = createFakeStagehand(page); + experimentalBatch.mockRejectedValueOnce( + new Error("Stagehand callback batch page was not found"), + ); + // First activePage() feeds the batch target; the second (inside the + // fallback) sees the tab gone. + context.activePage.mockResolvedValueOnce(page).mockResolvedValueOnce(undefined); + const tools = new StagehandFacadeTools(stagehand); + + await expect(tools.run(`return 42;`)).resolves.toBe(42); + expect(context.newPage).toHaveBeenCalledTimes(1); + expect(experimentalBatch).toHaveBeenCalledTimes(2); + expect(experimentalBatch.mock.calls[1][2]).toStrictEqual({ timeout: 60_000 }); + }); + + it("propagates other batch failures unchanged", async () => { + const { stagehand, experimentalBatch, context } = createFakeStagehand(createFakePage()); + experimentalBatch.mockRejectedValueOnce(new Error("extension disconnected")); + const tools = new StagehandFacadeTools(stagehand); + + await expect(tools.run(`return 1;`)).rejects.toThrow("extension disconnected"); + expect(context.newPage).not.toHaveBeenCalled(); + }); +}); From 3341bd6b99891c272067749990d2ef81d916606c Mon Sep 17 00:00:00 2001 From: miguel Date: Sun, 30 Aug 2026 21:16:06 -0700 Subject: [PATCH 02/49] feat(evals): stagehand_facade is the Playwright batch surface; add stagehand_facade_legacy stagehand_facade_legacy starts the same facade server with --surface=legacy and mounts LEGACY_FACADE_AGENT_INSTRUCTIONS so runs on the earlier prompt are never mixed with the Playwright-idiom surface. Registered on every harness that mounts the facade (claude_code, codex, mastra, pi, eve, deepagents, fx, cursor). --- packages/evals/core/contracts/tool.ts | 1 + packages/evals/core/tools/registry.ts | 6 ++- packages/evals/core/tools/stagehand_facade.ts | 54 ++++++++++++++++--- .../evals/framework/claudeCodeToolAdapter.ts | 4 +- packages/evals/framework/codexToolAdapter.ts | 5 +- packages/evals/framework/cursorToolAdapter.ts | 1 + packages/evals/framework/deepagentsRunner.ts | 2 +- .../evals/framework/deepagentsToolAdapter.ts | 1 + packages/evals/framework/eveToolAdapter.ts | 1 + packages/evals/framework/fxToolAdapter.ts | 3 +- packages/evals/framework/mastraToolAdapter.ts | 1 + packages/evals/framework/piToolAdapter.ts | 1 + .../evals/tests/core/stagehand-facade.test.ts | 51 +++++++++++++++++- .../tests/framework/benchHarness.test.ts | 4 ++ .../framework/claudeCodeToolAdapter.test.ts | 4 +- .../tests/framework/cursorToolAdapter.test.ts | 5 +- .../framework/deepagentsToolAdapter.test.ts | 2 +- .../tests/framework/eveToolAdapter.test.ts | 2 +- .../tests/framework/fxToolAdapter.test.ts | 9 +++- .../tests/framework/mastraToolAdapter.test.ts | 1 + .../framework/toolSurfaceResolution.test.ts | 2 +- 21 files changed, 139 insertions(+), 21 deletions(-) diff --git a/packages/evals/core/contracts/tool.ts b/packages/evals/core/contracts/tool.ts index 2971bb7662..ded28fab79 100644 --- a/packages/evals/core/contracts/tool.ts +++ b/packages/evals/core/contracts/tool.ts @@ -12,6 +12,7 @@ export type ToolSurface = | "playwright_mcp" | "chrome_devtools_mcp" | "stagehand_facade" + | "stagehand_facade_legacy" | "browse_cli"; export type StartupProfile = diff --git a/packages/evals/core/tools/registry.ts b/packages/evals/core/tools/registry.ts index 7cc36ecdd9..9c5ec117c1 100644 --- a/packages/evals/core/tools/registry.ts +++ b/packages/evals/core/tools/registry.ts @@ -5,12 +5,13 @@ import { ChromeDevtoolsMcpTool } from "./chrome_devtools_mcp.js"; import { PlaywrightCodeTool } from "./playwright_code.js"; import { PlaywrightMcpTool } from "./playwright_mcp.js"; import { StagehandCodeTool } from "./stagehand_code.js"; -import { StagehandFacadeTool } from "./stagehand_facade.js"; +import { StagehandFacadeLegacyTool, StagehandFacadeTool } from "./stagehand_facade.js"; import { UnderstudyCodeTool } from "./understudy_code.js"; /** Surfaces that exist only as an agent MCP mount; they have no runner-driven CoreSession (activePage() throws). */ export const AGENT_MOUNT_ONLY_TOOL_SURFACES: ReadonlySet = new Set([ "stagehand_facade", + "stagehand_facade_legacy", ]); export function isAgentMountOnlyToolSurface(toolSurface: ToolSurface): boolean { @@ -28,6 +29,7 @@ export function listCoreTools(): ToolSurface[] { // Listed here as part of the full enumeration, but agent-mount-only: // core-tier selection must use listCoreRunnableTools, which filters it. "stagehand_facade", + "stagehand_facade_legacy", "browse_cli", ]; } @@ -53,6 +55,8 @@ export function getCoreTool(toolSurface: ToolSurface): CoreTool { return new ChromeDevtoolsMcpTool(); case "stagehand_facade": return new StagehandFacadeTool(); + case "stagehand_facade_legacy": + return new StagehandFacadeLegacyTool(); case "browse_cli": return new BrowseCliTool(); default: diff --git a/packages/evals/core/tools/stagehand_facade.ts b/packages/evals/core/tools/stagehand_facade.ts index c65c4ac33d..3a4afb4deb 100644 --- a/packages/evals/core/tools/stagehand_facade.ts +++ b/packages/evals/core/tools/stagehand_facade.ts @@ -1,4 +1,7 @@ -import { FACADE_AGENT_INSTRUCTIONS } from "@browserbasehq/stagehand-integrations/facade"; +import { + FACADE_AGENT_INSTRUCTIONS, + LEGACY_FACADE_AGENT_INSTRUCTIONS, +} from "@browserbasehq/stagehand-integrations/facade"; import { buildAllowlistedEnv } from "@browserbasehq/stagehand-integrations/harness"; import { fileURLToPath } from "node:url"; import { EvalsError } from "../../errors.js"; @@ -11,6 +14,7 @@ import type { StartupProfile, ToolStartInput, ToolStartResult, + ToolSurface, } from "../contracts/tool.js"; import type { TargetKind } from "../contracts/targets.js"; import { startStagehandFacadeBridge } from "./stagehandFacadeBridge.js"; @@ -103,12 +107,22 @@ export function buildStagehandFacadeServerSpec(environment: ToolStartInput["envi }; } +/** Same facade server, started with `--surface=legacy` so it advertises the pre-Playwright-idiom run contract. */ +export function buildStagehandFacadeLegacyServerSpec(environment: ToolStartInput["environment"]): { + command: string; + args: string[]; + env: Record; +} { + const spec = buildStagehandFacadeServerSpec(environment); + return { ...spec, args: [...spec.args, "--surface=legacy"] }; +} + function connectionModeFromProfile(startupProfile: StartupProfile): ConnectionMode { return startupProfile === "tool_create_browserbase" ? "browserbase_native" : "launch"; } export class StagehandFacadeTool implements CoreTool { - readonly id = "stagehand_facade"; + readonly id: ToolSurface = "stagehand_facade"; readonly surface = "mcp"; readonly family = "stagehand"; readonly supportedStartupProfiles: StartupProfile[] = [ @@ -128,30 +142,40 @@ export class StagehandFacadeTool implements CoreTool { } = {}, ) {} + protected defaultServerSpec(environment: ToolStartInput["environment"]) { + return buildStagehandFacadeServerSpec(environment); + } + + protected promptInstructions(): string { + return FACADE_AGENT_INSTRUCTIONS; + } + async start(input: ToolStartInput): Promise { const expectedProfile = input.environment === "BROWSERBASE" ? "tool_create_browserbase" : "tool_launch_local"; if (input.startupProfile !== expectedProfile) { throw new StagehandFacadeToolError( - "stagehand_facade received an invalid startup profile for the selected environment.", + `${this.id} received an invalid startup profile for the selected environment.`, ); } const session = new StagehandFacadeMountSession(); - const spec = (this.options.serverSpec ?? buildStagehandFacadeServerSpec)(input.environment); + const spec = this.options.serverSpec + ? this.options.serverSpec(input.environment) + : this.defaultServerSpec(input.environment); const bridge = await startStagehandFacadeBridge({ server: spec, logger: input.logger }); if (typeof input.logger?.log === "function") { input.logger.log({ category: "stagehand_facade", level: 1, - message: `Started runner-owned stagehand_facade bridge on 127.0.0.1:${bridge.port}.`, + message: `Started runner-owned ${this.id} bridge on 127.0.0.1:${bridge.port}.`, }); } return { session, agentMount: { via: "mcp", - promptInstructions: FACADE_AGENT_INSTRUCTIONS, + promptInstructions: this.promptInstructions(), mcpServers: { stagehand: bridge.mcpServerSpec, }, @@ -171,3 +195,21 @@ export class StagehandFacadeTool implements CoreTool { }; } } + +/** + * The facade surface as it shipped before the Playwright-idiom prompt: same + * server, same snapshot/screenshot tools, but the earlier run description and + * agent instructions. Kept under its own id so trajectories from the two + * prompts are never compared as one surface. + */ +export class StagehandFacadeLegacyTool extends StagehandFacadeTool { + override readonly id: ToolSurface = "stagehand_facade_legacy"; + + protected override defaultServerSpec(environment: ToolStartInput["environment"]) { + return buildStagehandFacadeLegacyServerSpec(environment); + } + + protected override promptInstructions(): string { + return LEGACY_FACADE_AGENT_INSTRUCTIONS; + } +} diff --git a/packages/evals/framework/claudeCodeToolAdapter.ts b/packages/evals/framework/claudeCodeToolAdapter.ts index d0e7feb152..74a8fcf639 100644 --- a/packages/evals/framework/claudeCodeToolAdapter.ts +++ b/packages/evals/framework/claudeCodeToolAdapter.ts @@ -88,6 +88,7 @@ export const CLAUDE_CODE_TOOL_SURFACES: ToolSurface[] = [ "playwright_mcp", "chrome_devtools_mcp", "stagehand_facade", + "stagehand_facade_legacy", ]; // The CLI skill below is written for interactive use and covers surface @@ -196,7 +197,8 @@ export async function prepareClaudeCodeToolAdapter( case "stagehand_code": case "playwright_mcp": case "chrome_devtools_mcp": - case "stagehand_facade": { + case "stagehand_facade": + case "stagehand_facade_legacy": { return prepareMountedCoreToolAdapter({ ...input, toolSurface, diff --git a/packages/evals/framework/codexToolAdapter.ts b/packages/evals/framework/codexToolAdapter.ts index 461823e1c4..4bce0d6b88 100644 --- a/packages/evals/framework/codexToolAdapter.ts +++ b/packages/evals/framework/codexToolAdapter.ts @@ -60,6 +60,7 @@ export const CODEX_TOOL_SURFACES: ToolSurface[] = [ "playwright_mcp", "chrome_devtools_mcp", "stagehand_facade", + "stagehand_facade_legacy", ]; const STAGEHAND_FACADE_MCP_TIMEOUTS = { @@ -71,7 +72,9 @@ export function buildCodexMcpServers( toolSurface: ToolSurface, mcpServers: Record, ): Record { - if (toolSurface !== "stagehand_facade") return mcpServers; + if (toolSurface !== "stagehand_facade" && toolSurface !== "stagehand_facade_legacy") { + return mcpServers; + } return Object.fromEntries( Object.entries(mcpServers).map(([name, config]) => [ diff --git a/packages/evals/framework/cursorToolAdapter.ts b/packages/evals/framework/cursorToolAdapter.ts index 857fc06e6d..4fb6c3c610 100644 --- a/packages/evals/framework/cursorToolAdapter.ts +++ b/packages/evals/framework/cursorToolAdapter.ts @@ -35,6 +35,7 @@ export interface PreparedCursorToolAdapter { export const CURSOR_TOOL_SURFACES: ToolSurface[] = [ "stagehand_facade", + "stagehand_facade_legacy", "playwright_mcp", "chrome_devtools_mcp", ]; diff --git a/packages/evals/framework/deepagentsRunner.ts b/packages/evals/framework/deepagentsRunner.ts index 6cf3505a6d..8866fea8a6 100644 --- a/packages/evals/framework/deepagentsRunner.ts +++ b/packages/evals/framework/deepagentsRunner.ts @@ -58,7 +58,7 @@ valid only for the latest snapshot of the active page. Snapshot again after navi `; export function buildDeepagentsSystemPrompt(toolSurface?: ToolSurface): string { - if (toolSurface === "stagehand_facade") { + if (toolSurface === "stagehand_facade" || toolSurface === "stagehand_facade_legacy") { return `${DEEPAGENTS_SHARED_SYSTEM_PROMPT}\n${DEEPAGENTS_FACADE_SYSTEM_PROMPT}`; } const toolGuidance = diff --git a/packages/evals/framework/deepagentsToolAdapter.ts b/packages/evals/framework/deepagentsToolAdapter.ts index 9ad2336ae3..6621bd1e5b 100644 --- a/packages/evals/framework/deepagentsToolAdapter.ts +++ b/packages/evals/framework/deepagentsToolAdapter.ts @@ -35,6 +35,7 @@ export interface PreparedDeepagentsToolAdapter { export const DEEPAGENTS_TOOL_SURFACES: ToolSurface[] = [ "stagehand_facade", + "stagehand_facade_legacy", "playwright_mcp", "chrome_devtools_mcp", ]; diff --git a/packages/evals/framework/eveToolAdapter.ts b/packages/evals/framework/eveToolAdapter.ts index a3650a6cb3..f7e20f277b 100644 --- a/packages/evals/framework/eveToolAdapter.ts +++ b/packages/evals/framework/eveToolAdapter.ts @@ -51,6 +51,7 @@ export interface PreparedEveToolAdapter { export const EVE_TOOL_SURFACES: ToolSurface[] = [ "stagehand_facade", + "stagehand_facade_legacy", "playwright_mcp", "chrome_devtools_mcp", ]; diff --git a/packages/evals/framework/fxToolAdapter.ts b/packages/evals/framework/fxToolAdapter.ts index 8949d889e7..801cf0d9b8 100644 --- a/packages/evals/framework/fxToolAdapter.ts +++ b/packages/evals/framework/fxToolAdapter.ts @@ -46,6 +46,7 @@ type FxMcpServerSpec = { export const FX_TOOL_SURFACES: ToolSurface[] = [ "stagehand_facade", + "stagehand_facade_legacy", "playwright_mcp", "chrome_devtools_mcp", ]; @@ -242,7 +243,7 @@ export async function prepareFxToolAdapter( const pathEnv = process.env.PATH ?? ""; const mcpOptions: FxMcpOptions = { home, pathEnv, parentEnv: process.env }; const mcpToolNames: Record = {}; - if (toolSurface === "stagehand_facade") { + if (toolSurface === "stagehand_facade" || toolSurface === "stagehand_facade_legacy") { mcpToolNames.stagehand = ["run", "snapshot", "screenshot"]; } else { const listMcpToolNames = input.listMcpToolNames ?? defaultListMcpToolNames; diff --git a/packages/evals/framework/mastraToolAdapter.ts b/packages/evals/framework/mastraToolAdapter.ts index 42ca4ce08e..504f38e453 100644 --- a/packages/evals/framework/mastraToolAdapter.ts +++ b/packages/evals/framework/mastraToolAdapter.ts @@ -25,6 +25,7 @@ import { ObservationRecorder, type StepObservation } from "./observationRecorder export const MASTRA_RUN_TOOL_NAME = "stagehand_browser_run"; export const MASTRA_TOOL_SURFACES: ToolSurface[] = [ "stagehand_facade", + "stagehand_facade_legacy", "playwright_mcp", "chrome_devtools_mcp", "stagehand_code", diff --git a/packages/evals/framework/piToolAdapter.ts b/packages/evals/framework/piToolAdapter.ts index 87a6fe2d39..a4c0883bbc 100644 --- a/packages/evals/framework/piToolAdapter.ts +++ b/packages/evals/framework/piToolAdapter.ts @@ -24,6 +24,7 @@ import { ObservationRecorder, type StepObservation } from "./observationRecorder export const PI_TOOL_SURFACES: ToolSurface[] = [ "stagehand_facade", + "stagehand_facade_legacy", "playwright_mcp", "chrome_devtools_mcp", "stagehand_code", diff --git a/packages/evals/tests/core/stagehand-facade.test.ts b/packages/evals/tests/core/stagehand-facade.test.ts index c484a51085..8ebb319408 100644 --- a/packages/evals/tests/core/stagehand-facade.test.ts +++ b/packages/evals/tests/core/stagehand-facade.test.ts @@ -1,9 +1,14 @@ -import { FACADE_AGENT_INSTRUCTIONS } from "@browserbasehq/stagehand-integrations/facade"; +import { + FACADE_AGENT_INSTRUCTIONS, + LEGACY_FACADE_AGENT_INSTRUCTIONS, +} from "@browserbasehq/stagehand-integrations/facade"; import { afterEach, beforeEach, describe, expect, it } from "vitest"; import { getCoreTool, listCoreRunnableTools, listCoreTools } from "../../core/tools/registry.js"; import { buildStagehandFacadeEnv, + buildStagehandFacadeLegacyServerSpec, buildStagehandFacadeServerSpec, + StagehandFacadeLegacyTool, StagehandFacadeTool, StagehandFacadeToolError, } from "../../core/tools/stagehand_facade.js"; @@ -60,6 +65,25 @@ describe("stagehand facade tool surface", () => { expect(getCoreTool("stagehand_facade")).toBeInstanceOf(StagehandFacadeTool); }); + it("registers the legacy prompt surface as a distinct agent-mount-only tool", () => { + expect(listCoreTools()).toContain("stagehand_facade_legacy"); + expect(listCoreRunnableTools()).not.toContain("stagehand_facade_legacy"); + const tool = getCoreTool("stagehand_facade_legacy"); + expect(tool).toBeInstanceOf(StagehandFacadeLegacyTool); + expect(tool.id).toBe("stagehand_facade_legacy"); + expect(getCoreTool("stagehand_facade")).not.toBeInstanceOf(StagehandFacadeLegacyTool); + expect(getCoreTool("stagehand_facade").id).toBe("stagehand_facade"); + }); + + it("starts the legacy surface from the same server with --surface=legacy", () => { + const playwright = buildStagehandFacadeServerSpec("LOCAL"); + const legacy = buildStagehandFacadeLegacyServerSpec("LOCAL"); + expect(legacy.command).toBe(playwright.command); + expect(legacy.env).toEqual(playwright.env); + expect(legacy.args).toEqual([...playwright.args, "--surface=legacy"]); + expect(playwright.args).not.toContain("--surface=legacy"); + }); + it("uses typed, sanitized errors for invalid lifecycle operations", async () => { const tool = new StagehandFacadeTool(); await expect( @@ -89,6 +113,9 @@ describe("stagehand facade tool surface", () => { it("preserves facade MCP timeouts in the Codex config", () => { const server = { command: "node", args: ["stdio-server.mjs"] }; + expect(buildCodexMcpServers("stagehand_facade_legacy", { stagehand: server })).toEqual( + buildCodexMcpServers("stagehand_facade", { stagehand: server }), + ); expect(buildCodexMcpServers("stagehand_facade", { stagehand: server })).toEqual({ stagehand: { ...server, @@ -146,6 +173,28 @@ describe("stagehand facade tool surface", () => { } }); + it("mounts the legacy surface with the legacy agent instructions", async () => { + const running = await new StagehandFacadeLegacyTool({ + serverSpec: (environment) => ({ + command: process.execPath, + args: ["-e", MINIMAL_FACADE_SOURCE], + env: buildStagehandFacadeEnv(environment), + }), + }).start({ + logger: {} as EvalLogger, + environment: "LOCAL", + startupProfile: "tool_launch_local", + }); + try { + if (running.agentMount?.via !== "mcp") throw new Error("expected MCP mount"); + expect(running.agentMount.promptInstructions).toBe(LEGACY_FACADE_AGENT_INSTRUCTIONS); + expect(running.agentMount.promptInstructions).not.toBe(FACADE_AGENT_INSTRUCTIONS); + expect(Object.keys(running.agentMount.mcpServers)).toEqual(["stagehand"]); + } finally { + await running.cleanup(); + } + }); + it("builds the default facade server spec with the shipped entrypoint", () => { process.env.STAGEHAND_MODEL_NAME = "openai/gpt-5-mini"; process.env.BROWSERBASE_API_KEY = "browserbase-secret"; diff --git a/packages/evals/tests/framework/benchHarness.test.ts b/packages/evals/tests/framework/benchHarness.test.ts index a9d4d47465..b69a2c423f 100644 --- a/packages/evals/tests/framework/benchHarness.test.ts +++ b/packages/evals/tests/framework/benchHarness.test.ts @@ -121,6 +121,7 @@ describe("bench harness registry", () => { expect(harness.start).toBeUndefined(); expect(harness.supportedToolSurfaces).toEqual([ "stagehand_facade", + "stagehand_facade_legacy", "playwright_mcp", "chrome_devtools_mcp", ]); @@ -139,6 +140,7 @@ describe("bench harness registry", () => { expect(harness.start).toBeUndefined(); expect(harness.supportedToolSurfaces).toEqual([ "stagehand_facade", + "stagehand_facade_legacy", "playwright_mcp", "chrome_devtools_mcp", ]); @@ -156,6 +158,7 @@ describe("bench harness registry", () => { expect(harness.start).toBeUndefined(); expect(harness.supportedToolSurfaces).toEqual([ "stagehand_facade", + "stagehand_facade_legacy", "playwright_mcp", "chrome_devtools_mcp", ]); @@ -176,6 +179,7 @@ describe("bench harness registry", () => { expect(harness.supportedToolSurfaces).toEqual(CURSOR_TOOL_SURFACES); expect(harness.supportedToolSurfaces).toEqual([ "stagehand_facade", + "stagehand_facade_legacy", "playwright_mcp", "chrome_devtools_mcp", ]); diff --git a/packages/evals/tests/framework/claudeCodeToolAdapter.test.ts b/packages/evals/tests/framework/claudeCodeToolAdapter.test.ts index 5acd0d939a..e5272fda23 100644 --- a/packages/evals/tests/framework/claudeCodeToolAdapter.test.ts +++ b/packages/evals/tests/framework/claudeCodeToolAdapter.test.ts @@ -46,7 +46,7 @@ describe("claude code tool adapter resolution", () => { it("rejects unsupported Claude Code tool surfaces for now", () => { expect(() => resolveToolSurface(claudeCodeHarness, "understudy_code")).toThrow( - /Harness "claude_code" supports --tool browse_cli, playwright_code, cdp_code, stagehand_code, playwright_mcp, chrome_devtools_mcp, or stagehand_facade; received "understudy_code"/, + /Harness "claude_code" supports --tool browse_cli, playwright_code, cdp_code, stagehand_code, playwright_mcp, chrome_devtools_mcp, stagehand_facade, or stagehand_facade_legacy; received "understudy_code"/, ); }); @@ -78,7 +78,7 @@ describe("claude code tool adapter resolution", () => { "runner_provided_browserbase_cdp", ); expect(() => resolveToolSurface(codexHarness, "understudy_code")).toThrow( - /Harness "codex" supports --tool browse_cli, playwright_code, cdp_code, stagehand_code, playwright_mcp, chrome_devtools_mcp, or stagehand_facade; received "understudy_code"/, + /Harness "codex" supports --tool browse_cli, playwright_code, cdp_code, stagehand_code, playwright_mcp, chrome_devtools_mcp, stagehand_facade, or stagehand_facade_legacy; received "understudy_code"/, ); expect(resolveStartupProfile("browse_cli", "LOCAL")).toBe("tool_launch_local"); expect(resolveStartupProfile("browse_cli", "BROWSERBASE")).toBe("tool_create_browserbase"); diff --git a/packages/evals/tests/framework/cursorToolAdapter.test.ts b/packages/evals/tests/framework/cursorToolAdapter.test.ts index c32407a873..b1c6d7d6bf 100644 --- a/packages/evals/tests/framework/cursorToolAdapter.test.ts +++ b/packages/evals/tests/framework/cursorToolAdapter.test.ts @@ -21,6 +21,7 @@ describe("cursor tool adapter helpers", () => { const harness = { harness: "cursor", supportedToolSurfaces: CURSOR_TOOL_SURFACES }; expect(CURSOR_TOOL_SURFACES).toEqual([ "stagehand_facade", + "stagehand_facade_legacy", "playwright_mcp", "chrome_devtools_mcp", ]); @@ -29,10 +30,10 @@ describe("cursor tool adapter helpers", () => { expect(resolveToolSurface(harness, "playwright_mcp")).toBe("playwright_mcp"); expect(resolveToolSurface(harness, "chrome_devtools_mcp")).toBe("chrome_devtools_mcp"); expect(() => resolveToolSurface(harness, "browse_cli")).toThrow( - /stagehand_facade, playwright_mcp, or chrome_devtools_mcp.*browse_cli/, + /stagehand_facade, stagehand_facade_legacy, playwright_mcp, or chrome_devtools_mcp.*browse_cli/, ); expect(() => resolveToolSurface(harness, "stagehand_code")).toThrow( - /stagehand_facade, playwright_mcp, or chrome_devtools_mcp.*stagehand_code/, + /stagehand_facade, stagehand_facade_legacy, playwright_mcp, or chrome_devtools_mcp.*stagehand_code/, ); }); diff --git a/packages/evals/tests/framework/deepagentsToolAdapter.test.ts b/packages/evals/tests/framework/deepagentsToolAdapter.test.ts index e530e8ebcc..6c86a0e395 100644 --- a/packages/evals/tests/framework/deepagentsToolAdapter.test.ts +++ b/packages/evals/tests/framework/deepagentsToolAdapter.test.ts @@ -18,7 +18,7 @@ describe("Deep Agents tool adapter helpers", () => { expect(resolveToolSurface(definition, "playwright_mcp")).toBe("playwright_mcp"); expect(resolveToolSurface(definition, "chrome_devtools_mcp")).toBe("chrome_devtools_mcp"); expect(() => resolveToolSurface(definition, "browse_cli")).toThrow( - /Harness "deepagents" supports --tool stagehand_facade, playwright_mcp, or chrome_devtools_mcp; received "browse_cli"/, + /Harness "deepagents" supports --tool stagehand_facade, stagehand_facade_legacy, playwright_mcp, or chrome_devtools_mcp; received "browse_cli"/, ); expect(resolveStartupProfile("stagehand_facade", "LOCAL")).toBe("tool_launch_local"); expect(resolveStartupProfile("stagehand_facade", "BROWSERBASE")).toBe( diff --git a/packages/evals/tests/framework/eveToolAdapter.test.ts b/packages/evals/tests/framework/eveToolAdapter.test.ts index 75fadc8c53..34f456c1c7 100644 --- a/packages/evals/tests/framework/eveToolAdapter.test.ts +++ b/packages/evals/tests/framework/eveToolAdapter.test.ts @@ -155,6 +155,6 @@ describe("Eve tool adapter helpers", () => { { harness: "eve", supportedToolSurfaces: EVE_TOOL_SURFACES }, "browse_cli", ), - ).toThrow(/stagehand_facade, playwright_mcp, or chrome_devtools_mcp/); + ).toThrow(/stagehand_facade, stagehand_facade_legacy, playwright_mcp, or chrome_devtools_mcp/); }); }); diff --git a/packages/evals/tests/framework/fxToolAdapter.test.ts b/packages/evals/tests/framework/fxToolAdapter.test.ts index bb34a4f44b..ac1f517843 100644 --- a/packages/evals/tests/framework/fxToolAdapter.test.ts +++ b/packages/evals/tests/framework/fxToolAdapter.test.ts @@ -17,11 +17,16 @@ import { describe("fx tool adapter helpers", () => { it("resolves surfaces and startup profiles through the shared registry helpers", () => { - expect(FX_TOOL_SURFACES).toEqual(["stagehand_facade", "playwright_mcp", "chrome_devtools_mcp"]); + expect(FX_TOOL_SURFACES).toEqual([ + "stagehand_facade", + "stagehand_facade_legacy", + "playwright_mcp", + "chrome_devtools_mcp", + ]); expect(resolveToolSurface(fxHarness)).toBe("stagehand_facade"); expect(resolveToolSurface(fxHarness, "playwright_mcp")).toBe("playwright_mcp"); expect(() => resolveToolSurface(fxHarness, "browse_cli")).toThrow( - 'Harness "fx" supports --tool stagehand_facade, playwright_mcp, or chrome_devtools_mcp; received "browse_cli".', + 'Harness "fx" supports --tool stagehand_facade, stagehand_facade_legacy, playwright_mcp, or chrome_devtools_mcp; received "browse_cli".', ); expect(resolveStartupProfile("stagehand_facade", "LOCAL")).toBe("tool_launch_local"); expect(resolveStartupProfile("chrome_devtools_mcp", "BROWSERBASE")).toBe( diff --git a/packages/evals/tests/framework/mastraToolAdapter.test.ts b/packages/evals/tests/framework/mastraToolAdapter.test.ts index 879fd89364..6be21cb555 100644 --- a/packages/evals/tests/framework/mastraToolAdapter.test.ts +++ b/packages/evals/tests/framework/mastraToolAdapter.test.ts @@ -31,6 +31,7 @@ describe("Mastra tool adapter", () => { it("lists supported surfaces with stagehand_facade first", () => { expect(MASTRA_TOOL_SURFACES).toEqual([ "stagehand_facade", + "stagehand_facade_legacy", "playwright_mcp", "chrome_devtools_mcp", "stagehand_code", diff --git a/packages/evals/tests/framework/toolSurfaceResolution.test.ts b/packages/evals/tests/framework/toolSurfaceResolution.test.ts index 385ecd4b17..1ceb7181d6 100644 --- a/packages/evals/tests/framework/toolSurfaceResolution.test.ts +++ b/packages/evals/tests/framework/toolSurfaceResolution.test.ts @@ -19,7 +19,7 @@ describe("tool surface resolution", () => { it("rejects unsupported surfaces with the full supported list", () => { expect(() => resolveToolSurface(claudeCodeHarness, "understudy_code")).toThrow( - /Harness "claude_code" supports --tool browse_cli, playwright_code, cdp_code, stagehand_code, playwright_mcp, chrome_devtools_mcp, or stagehand_facade; received "understudy_code"/, + /Harness "claude_code" supports --tool browse_cli, playwright_code, cdp_code, stagehand_code, playwright_mcp, chrome_devtools_mcp, stagehand_facade, or stagehand_facade_legacy; received "understudy_code"/, ); }); }); From 8cb2314931561bea294de7c188fafcc096cb0f1f Mon Sep 17 00:00:00 2001 From: miguel Date: Sun, 30 Aug 2026 22:34:43 -0700 Subject: [PATCH 03/49] fix(evals): default the rubric verifier to a live judge model V3Evaluator's built-in default (google/gemini-2.5-flash) was retired on 2026-07-09; without EVAL_VERIFIER_MODEL every rubric criterion failed with "Fused judgment call failed" and whole runs were silently unscored. The verifier now defaults to google/gemini-3.5-flash; an explicit override still fails loudly when its provider key is missing. --- packages/evals/framework/verifierAdapter.ts | 22 +++++++++++++-------- packages/evals/tui/commands/verify.ts | 2 +- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/packages/evals/framework/verifierAdapter.ts b/packages/evals/framework/verifierAdapter.ts index d7f1a3d8cf..31d5473b89 100644 --- a/packages/evals/framework/verifierAdapter.ts +++ b/packages/evals/framework/verifierAdapter.ts @@ -18,21 +18,27 @@ import type { TaskResult } from "./types.js"; const VERIFIER_MODEL_ENV = "EVAL_VERIFIER_MODEL"; const KEYLESS_VERIFIER_PROVIDERS = new Set(["bedrock", "ollama"]); +/** + * V3Evaluator's built-in default (google/gemini-2.5-flash) was retired + * 2026-07-09; leaving it in place fails every rubric criterion silently + * ("Fused judgment call failed"), which scores whole runs as unscored. + */ +export const DEFAULT_VERIFIER_MODEL = "google/gemini-3.5-flash"; /** - * Build the shared rubric verifier. By default V3Evaluator keeps its existing - * model selection; EVAL_VERIFIER_MODEL makes the verifier independently - * selectable for external harnesses and normal Stagehand runs alike. + * Build the shared rubric verifier. EVAL_VERIFIER_MODEL makes the verifier + * independently selectable for external harnesses and normal Stagehand runs + * alike; otherwise DEFAULT_VERIFIER_MODEL applies. */ export function createVerifierEvaluator(v3: V3): V3Evaluator { - const modelName = process.env[VERIFIER_MODEL_ENV]?.trim(); - if (!modelName) { - return new V3Evaluator(v3, { backend: "verifier" }); - } + const explicitModel = process.env[VERIFIER_MODEL_ENV]?.trim(); + const modelName = explicitModel || DEFAULT_VERIFIER_MODEL; const provider = modelName.includes("/") ? modelName.slice(0, modelName.indexOf("/")) : undefined; const apiKey = loadApiKeyFromEnv(provider, () => {}); - if (!apiKey && !KEYLESS_VERIFIER_PROVIDERS.has(provider ?? "")) { + // Only an explicit override fails loudly on a missing key; the default lets + // V3Evaluator resolve credentials itself (tests and keyless environments). + if (explicitModel && !apiKey && !KEYLESS_VERIFIER_PROVIDERS.has(provider ?? "")) { throw new Error( `${VERIFIER_MODEL_ENV} is set to "${modelName}", but no API key was found for provider "${provider ?? "unknown"}".`, ); diff --git a/packages/evals/tui/commands/verify.ts b/packages/evals/tui/commands/verify.ts index 763ea745b4..6cfff8f68f 100644 --- a/packages/evals/tui/commands/verify.ts +++ b/packages/evals/tui/commands/verify.ts @@ -48,7 +48,7 @@ ${bold("evals verify")} ${dim("— re-score a saved trajectory offline")} ${cyan("Options")} --model Override the verifier LLM (default: V3Evaluator's default, - currently google/gemini-2.5-flash). + default google/gemini-3.5-flash). --label Label appended to the output filename (default: rescore-). File written to scores/result_