feat: Playwright-batch facade surface, HardBench audit, and harness fixes from the smoke gates

packages/evals/core/contracts/tool.ts

@@ -12,6 +12,7 @@ export type ToolSurface =
   | "playwright_mcp"
   | "chrome_devtools_mcp"
   | "stagehand_facade"
+  | "stagehand_facade_legacy"
   | "browse_cli";
 
 export type StartupProfile =
@@ -135,6 +136,14 @@ export interface ToolStartInput {
   };
 }
 
+/** Why a tool surface's browser became unusable mid-run. */
+export interface BrowserSessionLoss {
+  cause: string;
+  /** Tool call that first observed the loss. */
+  tool?: string;
+  at?: string;
+}
+
 export interface ToolStartResult {
   session: CoreSession;
   /**
@@ -149,6 +158,11 @@ export interface ToolStartResult {
    * Implementations must swallow per-field failures and must not throw.
    */
   captureEvidence?: () => Promise<ProbeEvidence>;
+  /**
+   * Set once the surface's browser is gone for the rest of the run. Harnesses
+   * read it after the agent finishes to classify the outcome.
+   */
+  browserSessionLoss?: () => BrowserSessionLoss | undefined;
   /** Releases the runtime; `captureEvidence` is invalid after this resolves. */
   cleanup: () => Promise<void>;
   metadata: {

packages/evals/core/tools/browserSessionLoss.ts

@@ -0,0 +1,39 @@
+import type { BrowserSessionLoss } from "../contracts/tool.js";
+
+/**
+ * Wire strings the facade stdio server emits when its browser session is gone.
+ * Source of truth: packages/integrations/core/src/facade/contract.ts
+ * (BROWSER_SESSION_LOST_ERROR_PREFIX, SESSION_LOST_TELEMETRY_PREFIX). Mirrored
+ * here because evals consumes the integrations package through its built dist,
+ * and a facade build is allowed to lag behind the runner.
+ */
+export const BROWSER_SESSION_LOST_ERROR_PREFIX = "Browser session lost (";
+export const SESSION_LOST_TELEMETRY_PREFIX = "stagehand_facade_session_lost ";
+
+export function isBrowserSessionLostError(message: string): boolean {
+  return message.startsWith(BROWSER_SESSION_LOST_ERROR_PREFIX);
+}
+
+/** Extracts the cause from "Browser session lost (<cause>). ..." */
+export function browserSessionLostCause(message: string): string | undefined {
+  if (!isBrowserSessionLostError(message)) return undefined;
+  return /^Browser session lost \((.*?)\)\./u.exec(message)?.[1] ?? message;
+}
+
+export function parseSessionLossTelemetry(line: string): BrowserSessionLoss | undefined {
+  if (!line.startsWith(SESSION_LOST_TELEMETRY_PREFIX)) return undefined;
+  try {
+    const parsed = JSON.parse(line.slice(SESSION_LOST_TELEMETRY_PREFIX.length)) as Record<
+      string,
+      unknown
+    >;
+    if (typeof parsed.cause !== "string") return undefined;
+    return {
+      cause: parsed.cause,
+      ...(typeof parsed.tool === "string" && { tool: parsed.tool }),
+      ...(typeof parsed.at === "string" && { at: parsed.at }),
+    };
+  } catch {
+    return undefined;
+  }
+}

packages/evals/core/tools/registry.ts

@@ -5,12 +5,13 @@ import { ChromeDevtoolsMcpTool } from "./chrome_devtools_mcp.js";
 import { PlaywrightCodeTool } from "./playwright_code.js";
 import { PlaywrightMcpTool } from "./playwright_mcp.js";
 import { StagehandCodeTool } from "./stagehand_code.js";
-import { StagehandFacadeTool } from "./stagehand_facade.js";
+import { StagehandFacadeLegacyTool, StagehandFacadeTool } from "./stagehand_facade.js";
 import { UnderstudyCodeTool } from "./understudy_code.js";
 
 /** Surfaces that exist only as an agent MCP mount; they have no runner-driven CoreSession (activePage() throws). */
 export const AGENT_MOUNT_ONLY_TOOL_SURFACES: ReadonlySet<ToolSurface> = new Set<ToolSurface>([
   "stagehand_facade",
+  "stagehand_facade_legacy",
 ]);
 
 export function isAgentMountOnlyToolSurface(toolSurface: ToolSurface): boolean {
@@ -28,6 +29,7 @@ export function listCoreTools(): ToolSurface[] {
     // Listed here as part of the full enumeration, but agent-mount-only:
     // core-tier selection must use listCoreRunnableTools, which filters it.
     "stagehand_facade",
+    "stagehand_facade_legacy",
     "browse_cli",
   ];
 }
@@ -53,6 +55,8 @@ export function getCoreTool(toolSurface: ToolSurface): CoreTool {
       return new ChromeDevtoolsMcpTool();
     case "stagehand_facade":
       return new StagehandFacadeTool();
+    case "stagehand_facade_legacy":
+      return new StagehandFacadeLegacyTool();
     case "browse_cli":
       return new BrowseCliTool();
     default:

packages/evals/core/tools/stagehandFacadeBridge.ts

@@ -1,13 +1,17 @@
 import { spawn } from "node:child_process";
 import net, { type AddressInfo, type Socket } from "node:net";
 import type { ProbeEvidence } from "stagehand-v3";
+import { SESSION_INFO_TOOL_NAME } from "@browserbasehq/stagehand-integrations/facade";
+import type { BrowserSessionLoss } from "../contracts/tool.js";
+import { browserSessionLostCause, parseSessionLossTelemetry } from "./browserSessionLoss.js";
 import { sanitizeErrorMessage } from "@browserbasehq/stagehand-integrations/harness";
 import type { EvalLogger } from "../../logger.js";
 
 export const STAGEHAND_FACADE_BRIDGE_PORT_ENV = "STAGEHAND_EVALS_FACADE_BRIDGE_PORT";
 
 const DEFAULT_REQUEST_TIMEOUT_MS = 20_000;
 const DEFAULT_SHUTDOWN_TIMEOUT_MS = 10_000;
+const SESSION_INFO_TIMEOUT_MS = 60_000;
 const HOST_ENV_KEYS = [
   "PATH",
   "HOME",
@@ -29,6 +33,11 @@ export interface StagehandFacadeBridgeInput {
   requestTimeoutMs?: number;
 }
 
+export interface FacadeBrowserSessionInfo {
+  provider: "browserbase" | "local";
+  sessionId?: string;
+}
+
 export interface StagehandFacadeBridge {
   /** Loopback port the relay connects to. */
   port: number;
@@ -38,10 +47,25 @@ export interface StagehandFacadeBridge {
   agentConnections(): number;
   /** Whether any agent tools/call request has passed through. */
   sawAgentToolCall(): boolean;
+  /**
+   * Set once the facade reported its browser session gone. Every tool call
+   * after that returns the same terminal error, so failures past this point
+   * are consequences of the loss, not agent mistakes.
+   */
+  browserSessionLoss(): BrowserSessionLoss | undefined;
   /** Raw JSON-RPC request from the runner side. */
-  call(method: string, params?: Record<string, unknown>): Promise<unknown>;
+  call(
+    method: string,
+    params?: Record<string, unknown>,
+    options?: { timeoutMs?: number },
+  ): Promise<unknown>;
   /** Best-effort terminal/step evidence from the shared browser. */
   captureEvidence(): Promise<ProbeEvidence>;
+  /**
+   * Launches the facade browser if it has not started yet and reports where it
+   * lives. Runner-side only; the agent never sees this tool.
+   */
+  sessionInfo(): Promise<FacadeBrowserSessionInfo>;
   /** Idempotently closes the relay and facade process. */
   close(): Promise<void>;
 }
@@ -112,6 +136,12 @@ function isToolError(result: unknown): boolean {
   return Boolean(result && typeof result === "object" && (result as { isError?: unknown }).isError);
 }
 
+function sessionLostCauseFromToolResult(result: unknown): string | undefined {
+  if (!isToolError(result)) return undefined;
+  const text = contentBlocks(result).find((block) => typeof block.text === "string")?.text;
+  return typeof text === "string" ? browserSessionLostCause(text) : undefined;
+}
+
 export async function startStagehandFacadeBridge(
   input: StagehandFacadeBridgeInput,
 ): Promise<StagehandFacadeBridge> {
@@ -144,6 +174,13 @@ export async function startStagehandFacadeBridge(
   let exitDescription = "";
   let initializeResult: unknown;
   let closePromise: Promise<void> | undefined;
+  let sessionLoss: BrowserSessionLoss | undefined;
+
+  const noteSessionLoss = (loss: BrowserSessionLoss) => {
+    if (sessionLoss) return;
+    sessionLoss = loss;
+    log(`Facade browser session lost: ${loss.cause}`);
+  };
 
   const rejectPending = (error: Error) => {
     for (const entry of pending.values()) {
@@ -158,6 +195,8 @@ export async function startStagehandFacadeBridge(
     stderrLines.push(line);
     if (stderrLines.length > 20) stderrLines.shift();
     log(line);
+    const loss = parseSessionLossTelemetry(line);
+    if (loss) noteSessionLoss(loss);
   };
 
   child.stderr.on("data", (chunk: Buffer | string) => {
@@ -213,6 +252,10 @@ export async function startStagehandFacadeBridge(
       return;
     }
 
+    // The stderr telemetry line is the primary signal; the terminal tool error
+    // covers a facade build that predates it.
+    const lostCause = sessionLostCauseFromToolResult(message.result);
+    if (lostCause) noteSessionLoss({ cause: lostCause });
     const socket = agentRequests.get(idKey(message.id));
     agentRequests.delete(idKey(message.id));
     if (socket?.writable) socket.write(`${rawLine}\n`);
@@ -319,7 +362,12 @@ export async function startStagehandFacadeBridge(
   }
   const port = (server.address() as AddressInfo).port;
 
-  const call = (method: string, params?: Record<string, unknown>): Promise<unknown> => {
+  const call = (
+    method: string,
+    params?: Record<string, unknown>,
+    options: { timeoutMs?: number } = {},
+  ): Promise<unknown> => {
+    const timeoutMs = options.timeoutMs ?? requestTimeoutMs;
     if (closed) return Promise.reject(new Error("Stagehand facade bridge is closed"));
     if (exited) {
       return Promise.reject(
@@ -330,10 +378,8 @@ export async function startStagehandFacadeBridge(
     return new Promise<unknown>((resolve, reject) => {
       const timer = setTimeout(() => {
         pending.delete(id);
-        reject(
-          new Error(`Stagehand facade request "${method}" timed out after ${requestTimeoutMs}ms`),
-        );
-      }, requestTimeoutMs);
+        reject(new Error(`Stagehand facade request "${method}" timed out after ${timeoutMs}ms`));
+      }, timeoutMs);
       pending.set(id, { resolve, reject, timer });
       const request = {
         jsonrpc: "2.0",
@@ -401,6 +447,28 @@ export async function startStagehandFacadeBridge(
     return evidence;
   };
 
+  const sessionInfo = async (): Promise<FacadeBrowserSessionInfo> => {
+    // A cold Browserbase launch can outlast the per-request default.
+    const result = await call(
+      "tools/call",
+      { name: SESSION_INFO_TOOL_NAME, arguments: {} },
+      { timeoutMs: Math.max(requestTimeoutMs, SESSION_INFO_TIMEOUT_MS) },
+    );
+    if (isToolError(result)) {
+      const text = contentBlocks(result).find((block) => typeof block.text === "string")?.text;
+      throw new Error(sanitizeErrorMessage(`Stagehand facade session_info failed: ${text ?? ""}`));
+    }
+    const text = contentBlocks(result).find(
+      (block) => block.type === "text" && typeof block.text === "string",
+    )?.text;
+    const parsed = JSON.parse(String(text ?? "{}")) as Record<string, unknown>;
+    return {
+      provider: parsed.provider === "browserbase" ? "browserbase" : "local",
+      ...(typeof parsed.sessionId === "string" &&
+        parsed.sessionId && { sessionId: parsed.sessionId }),
+    };
+  };
+
   const close = (): Promise<void> => {
     closePromise ??= (async () => {
       closed = true;
@@ -431,8 +499,10 @@ export async function startStagehandFacadeBridge(
     },
     agentConnections: () => sockets.size,
     sawAgentToolCall: () => toolCallSeen,
+    browserSessionLoss: () => sessionLoss,
     call,
     captureEvidence,
+    sessionInfo,
     close,
   };
 

packages/evals/core/tools/stagehand_code.ts

@@ -465,7 +465,7 @@ export class StagehandCodeTool implements CoreTool {
     input.logger.log({
       category: "stagehand_code",
       message: "Initialized stagehand_code Stagehand SDK runtime.",
-      level: 1,
+      level: 2,
       auxiliary: {
         startupProfile: { value: input.startupProfile, type: "string" },
         environment: { value: input.environment, type: "string" },
@@ -496,6 +496,8 @@ export class StagehandCodeTool implements CoreTool {
         browserOwnership: "tool",
         connectionMode: connectionModeFromProfile(input.startupProfile),
         startupProfile: input.startupProfile,
+        ...(sdk.sessionUrl && { browserbaseSessionUrl: sdk.sessionUrl }),
+        ...(sdk.debugUrl && { browserbaseDebugUrl: sdk.debugUrl }),
       },
     };
   }