Standardize eval policy, budgets, session metadata, traces, and cost accounting

packages/evals/docs/verifier-gates.md

@@ -20,10 +20,10 @@ Blocker wording is recorded as `blockerMentioned` on criterion diagnostics. It n
 
 These fields depend on the producing runner; this verifier layer forwards them but does not make every harness emit them:
 
-- Where supplied, `facade_tool_calls`, `facade_tool_call_failures` and `facade_tool_calls_after_session_lost` distinguish attempted browser work from repeated terminal failures. Missing counters are unknown, not measured zero. A graded pass with an explicit zero browser-call count is shown in the batch summary; with `EVAL_MAX_UNVERIFIABLE_CRITERIA` enabled, it fails the batch gate.
+- Where supplied, `facade_tool_calls` and `facade_tool_call_failures` count attempted and failed browser work. Missing counters are unknown, not measured zero. Run-level browser loss comes from runner-owned telemetry. Normalized steps do not provide trusted per-call loss attribution, so tool-output text cannot exclude failures or synthesize a count after session loss. A graded pass with an explicit zero browser-call count is shown in the batch summary; with `EVAL_MAX_UNVERIFIABLE_CRITERIA` enabled, it fails the batch gate.
 - Separate agent, evidence-capture and verifier wall times are available only when recorded by the producer.
 - Usage must be interpreted with the producer's presence marker and cache convention. Legacy runners may supply zero placeholders; without an explicit presence marker, zero does not establish measured usage. Historical Cursor CLI usage remains unreported.
-- Costs reported by the harness can be retained. A `cost_source` field, when supplied by a producer, distinguishes reported dollars from a catalog estimate (`computed`). This verifier layer does not compute estimates. Without provenance, cost origin is unavailable; unknown or subscription costs must not be inferred as zero.
+- A producer's `cost_source` distinguishes reported dollars from a catalog estimate (`computed`). Shared runner estimates use the dated catalog in `pricing/pricing.json`; they are not invoices. This verifier layer does not compute estimates. Without provenance, cost origin is unavailable; unknown, tier-dependent or subscription costs must not be inferred as zero.
 - `harnessImplementation` records adapter and SDK versions when supplied. Its absence means unknown implementation; historical labels are preserved.
 
 Use `VERIFIER_PERSIST_TRAJECTORIES=1` for reviewable evidence. HardBench's compatibility gate rejects verifier errors, uncertainty sentinels, missing criteria and self-report fallbacks before accepting a result. Offline transport checks establish integration compatibility; live rubric accuracy still requires the separately recorded live fixtures.

packages/evals/framework/agentToolRuntime.ts

@@ -3,6 +3,7 @@ import { prepareCoreBrowserTarget } from "../core/targets/index.js";
 import { getCoreTool } from "../core/tools/registry.js";
 import { EvalsError } from "../errors.js";
 import type { EvalLogger } from "../logger.js";
+import { browserSessionFromMetadata, type BrowserSessionInfo } from "./browserSession.js";
 
 export interface AgentToolRuntimeInput {
   toolSurface: ToolSurface;
@@ -13,6 +14,12 @@ export interface AgentToolRuntimeInput {
 
 export interface StartedAgentToolRuntime {
   running: ToolStartResult;
+  /**
+   * Browser behind the surface, whether the runner provided it (Browserbase
+   * CDP target) or the tool created it (facade, stagehand_code). Known before
+   * the agent starts so the session URL can head the task log.
+   */
+  browserSession: BrowserSessionInfo;
   /** Closes the tool-owned runtime, then the runner-owned browser target. */
   cleanup: () => Promise<void>;
 }
@@ -49,6 +56,10 @@ export async function startAgentToolRuntime(
   let cleanupPromise: Promise<void> | undefined;
   return {
     running,
+    browserSession: browserSessionFromMetadata(
+      { ...running.metadata, ...target.metadata },
+      input.environment,
+    ),
     cleanup: async () => {
       cleanupPromise ??= (async () => {
         try {

packages/evals/framework/benchHarness.ts

@@ -1,5 +1,6 @@
 import { V3, normalizeRubric, type AvailableModel, type TaskSpec } from "stagehand-v3";
 import { EvalsError } from "../errors.js";
+import { sanitizeErrorMessage } from "@browserbasehq/stagehand-integrations/harness";
 import type { EvalLogger } from "../logger.js";
 import type { StagehandInitResult } from "../initStagehand.js";
 import type { EvalInput } from "../types/evals.js";
@@ -26,7 +27,13 @@ import {
   buildExternalHarnessTaskPlan,
   type ExternalHarnessTaskPlan,
 } from "./externalHarnessPlan.js";
+import {
+  logBrowserSession,
+  withBrowserSession,
+  type BrowserSessionInfo,
+} from "./browserSession.js";
 import { withHarnessAgentSpan } from "./otel.js";
+import { verifierTraceEnabled } from "./verifierTrace.js";
 import type { DiscoveredTask, TaskResult } from "./types.js";
 import type { BenchMatrixRow, BenchTaskKind, Harness } from "./benchTypes.js";
 import { DEFAULT_BENCH_HARNESS } from "./benchTypes.js";
@@ -69,7 +76,7 @@ export interface BenchHarness {
   supportsApi: boolean;
   /**
    * Tool surfaces this harness can mount for the agent, in display order; the
-   * first entry is the default when --tool is omitted. An empty list means the
+   * facade is preferred when --tool is omitted, otherwise the first entry. An empty list means the
    * harness does not mount tool surfaces and the planner passes the requested
    * surface/profile through unchanged as row metadata (stagehand harness).
    */
@@ -105,7 +112,14 @@ export interface ExternalHarnessRunInput<TAdapter> {
   verifier: ExternalHarnessVerifierConfig;
 }
 
-export interface ExternalHarnessDefinition<TAdapter extends { cleanup: () => Promise<void> }> {
+/** What every prepared external-harness adapter must expose to the shared lifecycle. */
+export interface ExternalHarnessAdapterBase {
+  cleanup: () => Promise<void>;
+  /** Browser behind the mounted surface; logged before the agent starts. */
+  browserSession?: BrowserSessionInfo;
+}
+
+export interface ExternalHarnessDefinition<TAdapter extends ExternalHarnessAdapterBase> {
   harness: string;
   supportedToolSurfaces: ToolSurface[];
   defaultModels: AvailableModel[];
@@ -119,7 +133,7 @@ export interface ExternalHarnessDefinition<TAdapter extends { cleanup: () => Pro
  * Define the lifecycle common to external agent harnesses without registering
  * it; registry ownership stays explicit so list order remains deterministic.
  */
-export function defineExternalHarness<TAdapter extends { cleanup: () => Promise<void> }>(
+export function defineExternalHarness<TAdapter extends ExternalHarnessAdapterBase>(
   definition: ExternalHarnessDefinition<TAdapter>,
 ): BenchHarness {
   const {
@@ -148,6 +162,9 @@ export function defineExternalHarness<TAdapter extends { cleanup: () => Promise<
       // the adapter and the carrier.
       const carrierV3 = buildVerifierCarrierV3(logger);
       let toolAdapter: TAdapter | undefined;
+      let browserSession: BrowserSessionInfo = {
+        provider: row.config.environment === "BROWSERBASE" ? "browserbase" : "local",
+      };
       try {
         toolAdapter = await prepareToolAdapter({
           toolSurface: row.config.toolSurface,
@@ -157,7 +174,9 @@ export function defineExternalHarness<TAdapter extends { cleanup: () => Promise<
           logger,
         });
         const preparedAdapter = toolAdapter;
-        return await withHarnessAgentSpan(
+        browserSession = preparedAdapter.browserSession ?? browserSession;
+        logBrowserSession(logger, browserSession);
+        const result = await withHarnessAgentSpan(
           {
             harness,
             model: input.modelName,
@@ -178,6 +197,18 @@ export function defineExternalHarness<TAdapter extends { cleanup: () => Promise<
               },
             }),
         );
+        return withBrowserSession(result, browserSession);
+      } catch (error) {
+        return withBrowserSession(
+          {
+            _success: false,
+            error: sanitizeErrorMessage(error instanceof Error ? error.message : String(error)),
+            harnessStatus: "sdk_error",
+            terminationReason: "sdk_error",
+            logs: logger.getLogs(),
+          },
+          browserSession,
+        );
       } finally {
         try {
           await toolAdapter?.cleanup();
@@ -208,7 +239,9 @@ function buildVerifierCarrierV3(logger: EvalLogger): V3 {
     disablePino: true,
     disableAPI: true,
     experimental: true,
-    verbose: 0,
+    // verbose 2 surfaces the judge's LLM request/response lines (level 2),
+    // which verifierAdapter routes to scores/verifier-trace.jsonl.
+    verbose: verifierTraceEnabled() ? 2 : 0,
   });
 }
 

packages/evals/framework/benchPlanner.ts

@@ -1,5 +1,6 @@
 import type { AvailableModel } from "stagehand-v3";
 import { EvalsError } from "../errors.js";
+import { explicitSnapshotActionsEnabled } from "@browserbasehq/stagehand-integrations/facade";
 import { buildOnlineMind2WebTestcases } from "../suites/onlineMind2Web.js";
 import { buildHardBenchmarkTestcases } from "../suites/hardbenchmark.js";
 import { buildWebTailBenchTestcases } from "../suites/webtailbench.js";
@@ -366,11 +367,15 @@ function withBenchMetadata(
 }
 
 function buildToolMetadata(row: BenchMatrixRow): Partial<Testcase["metadata"]> {
+  const promptVariant =
+    row.toolSurface === "stagehand_facade" && explicitSnapshotActionsEnabled()
+      ? { promptVariant: "explicit_snapshot_actions" }
+      : {};
   if (
     getBenchHarness(row.harness).supportedToolSurfaces.includes("browse_cli") &&
     row.toolSurface === "browse_cli"
   ) {
-    return getBrowseCliToolMetadata();
+    return { ...getBrowseCliToolMetadata(), ...promptVariant };
   }
-  return {};
+  return promptVariant;
 }

packages/evals/framework/browserSession.ts

@@ -0,0 +1,89 @@
+import type { LogLine } from "stagehand-v3";
+import type { TaskResult } from "./types.js";
+
+export const BROWSER_SESSION_LOG_CATEGORY = "session";
+
+/** Where the browser behind a run lives, resolved before the agent starts. */
+export interface BrowserSessionInfo {
+  provider: "browserbase" | "local";
+  sessionId?: string;
+  sessionUrl?: string;
+  debugUrl?: string;
+}
+
+export function browserbaseSessionUrl(sessionId: string): string {
+  return `https://www.browserbase.com/sessions/${encodeURIComponent(sessionId)}`;
+}
+
+/**
+ * Read the session fields core tools and runner-provided targets publish on
+ * their `metadata` (`browserbaseSessionId` / `browserbaseSessionUrl` /
+ * `browserbaseDebugUrl`). Falls back to the bare provider when a Browserbase
+ * surface does not report its session id (browse_cli).
+ */
+export function browserSessionFromMetadata(
+  metadata: Record<string, unknown> | undefined,
+  environment: "LOCAL" | "BROWSERBASE",
+): BrowserSessionInfo {
+  if (environment !== "BROWSERBASE") return { provider: "local" };
+  const rawUrl = readString(metadata?.browserbaseSessionUrl);
+  const sessionId =
+    readString(metadata?.browserbaseSessionId) ?? rawUrl?.match(/\/sessions\/([^/?#]+)/u)?.[1];
+  const sessionUrl = rawUrl ?? (sessionId ? browserbaseSessionUrl(sessionId) : undefined);
+  const debugUrl = readString(metadata?.browserbaseDebugUrl);
+  return {
+    provider: "browserbase",
+    ...(sessionId && { sessionId }),
+    ...(sessionUrl && { sessionUrl }),
+    ...(debugUrl && { debugUrl }),
+  };
+}
+
+export function formatBrowserSessionMessage(info: BrowserSessionInfo): string {
+  if (info.provider === "local") return "Browser: local";
+  if (!info.sessionUrl) return "Browser: browserbase (session id not reported by this surface)";
+  return `Browserbase session: ${info.sessionUrl}`;
+}
+
+/** Level-0 lines so the session pointer survives every log filter. */
+export function buildBrowserSessionLogLines(info: BrowserSessionInfo): LogLine[] {
+  const lines: LogLine[] = [
+    {
+      category: BROWSER_SESSION_LOG_CATEGORY,
+      level: 0,
+      message: formatBrowserSessionMessage(info),
+      auxiliary: {
+        provider: { value: info.provider, type: "string" },
+        ...(info.sessionId && { sessionId: { value: info.sessionId, type: "string" } }),
+        ...(info.sessionUrl && { sessionUrl: { value: info.sessionUrl, type: "string" } }),
+      },
+    },
+  ];
+  if (info.debugUrl) {
+    lines.push({
+      category: BROWSER_SESSION_LOG_CATEGORY,
+      level: 0,
+      message: `Browserbase debugger: ${info.debugUrl}`,
+    });
+  }
+  return lines;
+}
+
+export function logBrowserSession(sink: { log(line: LogLine): void }, info: BrowserSessionInfo) {
+  for (const line of buildBrowserSessionLogLines(info)) sink.log(line);
+}
+
+/** Surface the session on the TaskResult row so Braintrust output is filterable. */
+export function withBrowserSession(result: TaskResult, info: BrowserSessionInfo): TaskResult {
+  return {
+    ...result,
+    browserProvider: info.provider,
+    ...(info.sessionId && { browserbaseSessionId: info.sessionId }),
+    ...(info.sessionUrl && { sessionUrl: result.sessionUrl || info.sessionUrl }),
+    ...(info.debugUrl && { debugUrl: result.debugUrl || info.debugUrl }),
+  };
+}
+
+function readString(value: unknown): string | undefined {
+  return typeof value === "string" && value.trim() ? value : undefined;
+}