diff --git a/packages/evals/docs/verifier-gates.md b/packages/evals/docs/verifier-gates.md index 682592a99..187a5ce9b 100644 --- a/packages/evals/docs/verifier-gates.md +++ b/packages/evals/docs/verifier-gates.md @@ -20,10 +20,10 @@ Blocker wording is recorded as `blockerMentioned` on criterion diagnostics. It n These fields depend on the producing runner; this verifier layer forwards them but does not make every harness emit them: -- Where supplied, `facade_tool_calls`, `facade_tool_call_failures` and `facade_tool_calls_after_session_lost` distinguish attempted browser work from repeated terminal failures. Missing counters are unknown, not measured zero. A graded pass with an explicit zero browser-call count is shown in the batch summary; with `EVAL_MAX_UNVERIFIABLE_CRITERIA` enabled, it fails the batch gate. +- Where supplied, `facade_tool_calls` and `facade_tool_call_failures` count attempted and failed browser work. Missing counters are unknown, not measured zero. Run-level browser loss comes from runner-owned telemetry. Normalized steps do not provide trusted per-call loss attribution, so tool-output text cannot exclude failures or synthesize a count after session loss. A graded pass with an explicit zero browser-call count is shown in the batch summary; with `EVAL_MAX_UNVERIFIABLE_CRITERIA` enabled, it fails the batch gate. - Separate agent, evidence-capture and verifier wall times are available only when recorded by the producer. - Usage must be interpreted with the producer's presence marker and cache convention. Legacy runners may supply zero placeholders; without an explicit presence marker, zero does not establish measured usage. Historical Cursor CLI usage remains unreported. -- Costs reported by the harness can be retained. A `cost_source` field, when supplied by a producer, distinguishes reported dollars from a catalog estimate (`computed`). This verifier layer does not compute estimates. Without provenance, cost origin is unavailable; unknown or subscription costs must not be inferred as zero. +- A producer's `cost_source` distinguishes reported dollars from a catalog estimate (`computed`). Shared runner estimates use the dated catalog in `pricing/pricing.json`; they are not invoices. This verifier layer does not compute estimates. Without provenance, cost origin is unavailable; unknown, tier-dependent or subscription costs must not be inferred as zero. - `harnessImplementation` records adapter and SDK versions when supplied. Its absence means unknown implementation; historical labels are preserved. Use `VERIFIER_PERSIST_TRAJECTORIES=1` for reviewable evidence. HardBench's compatibility gate rejects verifier errors, uncertainty sentinels, missing criteria and self-report fallbacks before accepting a result. Offline transport checks establish integration compatibility; live rubric accuracy still requires the separately recorded live fixtures. diff --git a/packages/evals/framework/agentToolRuntime.ts b/packages/evals/framework/agentToolRuntime.ts index b5d41f4e0..c786c441d 100644 --- a/packages/evals/framework/agentToolRuntime.ts +++ b/packages/evals/framework/agentToolRuntime.ts @@ -3,6 +3,7 @@ import { prepareCoreBrowserTarget } from "../core/targets/index.js"; import { getCoreTool } from "../core/tools/registry.js"; import { EvalsError } from "../errors.js"; import type { EvalLogger } from "../logger.js"; +import { browserSessionFromMetadata, type BrowserSessionInfo } from "./browserSession.js"; export interface AgentToolRuntimeInput { toolSurface: ToolSurface; @@ -13,6 +14,12 @@ export interface AgentToolRuntimeInput { export interface StartedAgentToolRuntime { running: ToolStartResult; + /** + * Browser behind the surface, whether the runner provided it (Browserbase + * CDP target) or the tool created it (facade, stagehand_code). Known before + * the agent starts so the session URL can head the task log. + */ + browserSession: BrowserSessionInfo; /** Closes the tool-owned runtime, then the runner-owned browser target. */ cleanup: () => Promise; } @@ -49,6 +56,10 @@ export async function startAgentToolRuntime( let cleanupPromise: Promise | undefined; return { running, + browserSession: browserSessionFromMetadata( + { ...running.metadata, ...target.metadata }, + input.environment, + ), cleanup: async () => { cleanupPromise ??= (async () => { try { diff --git a/packages/evals/framework/benchHarness.ts b/packages/evals/framework/benchHarness.ts index 99c8f3d61..71f9766fd 100644 --- a/packages/evals/framework/benchHarness.ts +++ b/packages/evals/framework/benchHarness.ts @@ -1,5 +1,6 @@ import { V3, normalizeRubric, type AvailableModel, type TaskSpec } from "stagehand-v3"; import { EvalsError } from "../errors.js"; +import { sanitizeErrorMessage } from "@browserbasehq/stagehand-integrations/harness"; import type { EvalLogger } from "../logger.js"; import type { StagehandInitResult } from "../initStagehand.js"; import type { EvalInput } from "../types/evals.js"; @@ -26,7 +27,13 @@ import { buildExternalHarnessTaskPlan, type ExternalHarnessTaskPlan, } from "./externalHarnessPlan.js"; +import { + logBrowserSession, + withBrowserSession, + type BrowserSessionInfo, +} from "./browserSession.js"; import { withHarnessAgentSpan } from "./otel.js"; +import { verifierTraceEnabled } from "./verifierTrace.js"; import type { DiscoveredTask, TaskResult } from "./types.js"; import type { BenchMatrixRow, BenchTaskKind, Harness } from "./benchTypes.js"; import { DEFAULT_BENCH_HARNESS } from "./benchTypes.js"; @@ -69,7 +76,7 @@ export interface BenchHarness { supportsApi: boolean; /** * Tool surfaces this harness can mount for the agent, in display order; the - * first entry is the default when --tool is omitted. An empty list means the + * facade is preferred when --tool is omitted, otherwise the first entry. An empty list means the * harness does not mount tool surfaces and the planner passes the requested * surface/profile through unchanged as row metadata (stagehand harness). */ @@ -105,7 +112,14 @@ export interface ExternalHarnessRunInput { verifier: ExternalHarnessVerifierConfig; } -export interface ExternalHarnessDefinition Promise }> { +/** What every prepared external-harness adapter must expose to the shared lifecycle. */ +export interface ExternalHarnessAdapterBase { + cleanup: () => Promise; + /** Browser behind the mounted surface; logged before the agent starts. */ + browserSession?: BrowserSessionInfo; +} + +export interface ExternalHarnessDefinition { harness: string; supportedToolSurfaces: ToolSurface[]; defaultModels: AvailableModel[]; @@ -119,7 +133,7 @@ export interface ExternalHarnessDefinition Pro * Define the lifecycle common to external agent harnesses without registering * it; registry ownership stays explicit so list order remains deterministic. */ -export function defineExternalHarness Promise }>( +export function defineExternalHarness( definition: ExternalHarnessDefinition, ): BenchHarness { const { @@ -148,6 +162,9 @@ export function defineExternalHarness Promise< // the adapter and the carrier. const carrierV3 = buildVerifierCarrierV3(logger); let toolAdapter: TAdapter | undefined; + let browserSession: BrowserSessionInfo = { + provider: row.config.environment === "BROWSERBASE" ? "browserbase" : "local", + }; try { toolAdapter = await prepareToolAdapter({ toolSurface: row.config.toolSurface, @@ -157,7 +174,9 @@ export function defineExternalHarness Promise< logger, }); const preparedAdapter = toolAdapter; - return await withHarnessAgentSpan( + browserSession = preparedAdapter.browserSession ?? browserSession; + logBrowserSession(logger, browserSession); + const result = await withHarnessAgentSpan( { harness, model: input.modelName, @@ -178,6 +197,18 @@ export function defineExternalHarness Promise< }, }), ); + return withBrowserSession(result, browserSession); + } catch (error) { + return withBrowserSession( + { + _success: false, + error: sanitizeErrorMessage(error instanceof Error ? error.message : String(error)), + harnessStatus: "sdk_error", + terminationReason: "sdk_error", + logs: logger.getLogs(), + }, + browserSession, + ); } finally { try { await toolAdapter?.cleanup(); @@ -208,7 +239,9 @@ function buildVerifierCarrierV3(logger: EvalLogger): V3 { disablePino: true, disableAPI: true, experimental: true, - verbose: 0, + // verbose 2 surfaces the judge's LLM request/response lines (level 2), + // which verifierAdapter routes to scores/verifier-trace.jsonl. + verbose: verifierTraceEnabled() ? 2 : 0, }); } diff --git a/packages/evals/framework/benchPlanner.ts b/packages/evals/framework/benchPlanner.ts index ad35a6f79..595ab6023 100644 --- a/packages/evals/framework/benchPlanner.ts +++ b/packages/evals/framework/benchPlanner.ts @@ -1,5 +1,6 @@ import type { AvailableModel } from "stagehand-v3"; import { EvalsError } from "../errors.js"; +import { explicitSnapshotActionsEnabled } from "@browserbasehq/stagehand-integrations/facade"; import { buildOnlineMind2WebTestcases } from "../suites/onlineMind2Web.js"; import { buildHardBenchmarkTestcases } from "../suites/hardbenchmark.js"; import { buildWebTailBenchTestcases } from "../suites/webtailbench.js"; @@ -366,11 +367,15 @@ function withBenchMetadata( } function buildToolMetadata(row: BenchMatrixRow): Partial { + const promptVariant = + row.toolSurface === "stagehand_facade" && explicitSnapshotActionsEnabled() + ? { promptVariant: "explicit_snapshot_actions" } + : {}; if ( getBenchHarness(row.harness).supportedToolSurfaces.includes("browse_cli") && row.toolSurface === "browse_cli" ) { - return getBrowseCliToolMetadata(); + return { ...getBrowseCliToolMetadata(), ...promptVariant }; } - return {}; + return promptVariant; } diff --git a/packages/evals/framework/browserSession.ts b/packages/evals/framework/browserSession.ts new file mode 100644 index 000000000..855fd80f3 --- /dev/null +++ b/packages/evals/framework/browserSession.ts @@ -0,0 +1,89 @@ +import type { LogLine } from "stagehand-v3"; +import type { TaskResult } from "./types.js"; + +export const BROWSER_SESSION_LOG_CATEGORY = "session"; + +/** Where the browser behind a run lives, resolved before the agent starts. */ +export interface BrowserSessionInfo { + provider: "browserbase" | "local"; + sessionId?: string; + sessionUrl?: string; + debugUrl?: string; +} + +export function browserbaseSessionUrl(sessionId: string): string { + return `https://www.browserbase.com/sessions/${encodeURIComponent(sessionId)}`; +} + +/** + * Read the session fields core tools and runner-provided targets publish on + * their `metadata` (`browserbaseSessionId` / `browserbaseSessionUrl` / + * `browserbaseDebugUrl`). Falls back to the bare provider when a Browserbase + * surface does not report its session id (browse_cli). + */ +export function browserSessionFromMetadata( + metadata: Record | undefined, + environment: "LOCAL" | "BROWSERBASE", +): BrowserSessionInfo { + if (environment !== "BROWSERBASE") return { provider: "local" }; + const rawUrl = readString(metadata?.browserbaseSessionUrl); + const sessionId = + readString(metadata?.browserbaseSessionId) ?? rawUrl?.match(/\/sessions\/([^/?#]+)/u)?.[1]; + const sessionUrl = rawUrl ?? (sessionId ? browserbaseSessionUrl(sessionId) : undefined); + const debugUrl = readString(metadata?.browserbaseDebugUrl); + return { + provider: "browserbase", + ...(sessionId && { sessionId }), + ...(sessionUrl && { sessionUrl }), + ...(debugUrl && { debugUrl }), + }; +} + +export function formatBrowserSessionMessage(info: BrowserSessionInfo): string { + if (info.provider === "local") return "Browser: local"; + if (!info.sessionUrl) return "Browser: browserbase (session id not reported by this surface)"; + return `Browserbase session: ${info.sessionUrl}`; +} + +/** Level-0 lines so the session pointer survives every log filter. */ +export function buildBrowserSessionLogLines(info: BrowserSessionInfo): LogLine[] { + const lines: LogLine[] = [ + { + category: BROWSER_SESSION_LOG_CATEGORY, + level: 0, + message: formatBrowserSessionMessage(info), + auxiliary: { + provider: { value: info.provider, type: "string" }, + ...(info.sessionId && { sessionId: { value: info.sessionId, type: "string" } }), + ...(info.sessionUrl && { sessionUrl: { value: info.sessionUrl, type: "string" } }), + }, + }, + ]; + if (info.debugUrl) { + lines.push({ + category: BROWSER_SESSION_LOG_CATEGORY, + level: 0, + message: `Browserbase debugger: ${info.debugUrl}`, + }); + } + return lines; +} + +export function logBrowserSession(sink: { log(line: LogLine): void }, info: BrowserSessionInfo) { + for (const line of buildBrowserSessionLogLines(info)) sink.log(line); +} + +/** Surface the session on the TaskResult row so Braintrust output is filterable. */ +export function withBrowserSession(result: TaskResult, info: BrowserSessionInfo): TaskResult { + return { + ...result, + browserProvider: info.provider, + ...(info.sessionId && { browserbaseSessionId: info.sessionId }), + ...(info.sessionUrl && { sessionUrl: result.sessionUrl || info.sessionUrl }), + ...(info.debugUrl && { debugUrl: result.debugUrl || info.debugUrl }), + }; +} + +function readString(value: unknown): string | undefined { + return typeof value === "string" && value.trim() ? value : undefined; +} diff --git a/packages/evals/framework/costEstimate.ts b/packages/evals/framework/costEstimate.ts new file mode 100644 index 000000000..afbf56598 --- /dev/null +++ b/packages/evals/framework/costEstimate.ts @@ -0,0 +1,273 @@ +import fs from "node:fs"; +import path from "node:path"; +import { getPackageRootDir } from "../runtimePaths.js"; +import type { NormalizedUsage } from "./usageNormalization.js"; + +/** USD per million tokens. `null` marks a model the owner has not priced yet. */ +export interface ModelPrice { + input_per_m: number | null; + cached_input_per_m: number | null; + /** Cache-write rate where the provider bills one; falls back to `input_per_m`. */ + cache_write_input_per_m?: number | null; + output_per_m: number | null; + source: string; + note?: string; +} + +export interface PriceMap { + as_of: string; + models: Record; +} + +/** + * Where `cost_usd` came from: + * - `reported`: the harness's own billing channel reported dollars. + * - `computed`: the harness called the provider API directly with our key, so + * the estimate is normalized tokens × the dated catalog price in pricing.json. + * - `unavailable`: neither — subscription-billed cells (cursor, claude_code on + * a plan) or a model missing from the price map. No cost metric is emitted; + * token efficiency stays on the usage_* metrics. + */ +export type CostSource = "reported" | "computed" | "unavailable"; + +export interface BilledCost { + cost_usd?: number; + cost_source: CostSource; + /** Who billed the tokens, e.g. "anthropic_api", "ai_gateway", "pi_catalog", "subscription". */ + billing_channel: string; + /** Catalog provenance for a list-price estimate, never for a reported bill. */ + cost_pricing?: { as_of: string; model: string; source: string }; +} + +/** + * Billing channel per harness. + * + * | harness | reports dollars? | channel when reported | when not reported | + * |-------------|-----------------------------------------|-----------------------|-------------------------------------------------------| + * | claude_code | total_cost_usd on the result message | anthropic_api | subscription (Claude plan; no dollars, unavailable) | + * | eve | costUsd per step (gateway-routed models)| ai_gateway | first-party creators run direct → computed

_api | + * | pi | usage.cost.total from pi's model catalog| pi_catalog | direct provider call → computed _api | + * | fx | total_cost in usage-v2.json | fx_gateway | fx always bills via its gateway → unavailable | + * | codex | never (turn.completed has tokens only) | — | OpenAI API with our key → computed openai_api | + * | mastra | never (AI SDK usage has no dollars) | — | provider SDK with our key → computed _api | + * | deepagents | never (LangChain usage_metadata) | — | provider SDK with our key → computed _api | + * | cursor | never | — | subscription → unavailable | + */ +const REPORTED_CHANNEL: Readonly> = { + claude_code: "anthropic_api", + claude_cua: "anthropic_api", + gemini_cua: "google_api", + eve: "ai_gateway", + pi: "pi_catalog", + fx: "fx_gateway", +}; + +/** Harnesses whose unreported bill is our own provider-API spend, priceable at list. */ +const DIRECT_PROVIDER_HARNESSES: ReadonlySet = new Set([ + "codex", + "claude_cua", + "gemini_cua", + "mastra", + "deepagents", + "eve", + "pi", +]); + +const SUBSCRIPTION_HARNESSES: ReadonlySet = new Set(["cursor", "claude_code"]); + +export interface ResolveBilledCostInput { + harness: string; + model: string | undefined; + usage: NormalizedUsage; + /** Dollars the harness's own channel reported for the run, when any. */ + reportedCostUsd?: number; + priceMap?: PriceMap; +} + +/** + * The cost column: reported dollars win; + * otherwise a direct-provider harness is estimated at catalog price; otherwise the + * cost is unavailable rather than zero. + */ +export function resolveBilledCost({ + harness, + model, + usage, + reportedCostUsd, + priceMap = loadPriceMap(), +}: ResolveBilledCostInput): BilledCost { + if ( + typeof reportedCostUsd === "number" && + Number.isFinite(reportedCostUsd) && + reportedCostUsd >= 0 + ) { + return { + cost_usd: reportedCostUsd, + cost_source: "reported", + billing_channel: REPORTED_CHANNEL[harness] ?? `${harness}_reported`, + }; + } + const provider = providerOf(model); + const channel = + harness === "fx" + ? "fx_gateway" + : SUBSCRIPTION_HARNESSES.has(harness) + ? "subscription" + : provider + ? `${provider}_api` + : "none"; + if (!DIRECT_PROVIDER_HARNESSES.has(harness) || usage.convention === "unreported") { + return { cost_source: "unavailable", billing_channel: channel }; + } + const matched = resolveModelPrice(model, priceMap); + const computed = computeListCost(usage, model, priceMap); + return computed === undefined || !matched + ? { cost_source: "unavailable", billing_channel: channel } + : { + cost_usd: computed, + cost_source: "computed", + billing_channel: channel, + cost_pricing: { + as_of: priceMap.as_of, + model: matched.key, + source: matched.price.source, + }, + }; +} + +/** Provider segment of a configured model id, normalized to the price-map spelling. */ +export function providerOf(model: string | undefined): string | undefined { + if (!model) return undefined; + const [candidate] = modelPriceCandidates(model); + if (!candidate?.includes("/")) return undefined; + const provider = candidate.slice(0, candidate.indexOf("/")); + return provider === "spacexai" || provider === "x-ai" ? "xai" : provider; +} + +/** + * cost = uncached·p_in + cached·p_cached + cache_write·(p_write ?? p_in) + output·p_out, + * plus reasoning at the output rate only when the SDK reports it outside output. + */ +export function computeListCost( + usage: NormalizedUsage, + model: string | undefined, + priceMap: PriceMap = loadPriceMap(), +): number | undefined { + if (usage.convention === "unreported") return undefined; + const price = resolveModelPrice(model, priceMap)?.price; + if (!price) return undefined; + const cacheWriteRate = price.cache_write_input_per_m ?? price.input_per_m!; + const billedOutput = usage.output + (usage.reasoning_in_output ? 0 : usage.reasoning); + const cost = + (usage.input_uncached * price.input_per_m! + + usage.input_cached * price.cached_input_per_m! + + usage.input_cache_write * cacheWriteRate + + billedOutput * price.output_per_m!) / + 1_000_000; + return Number.isFinite(cost) && cost >= 0 ? Number(cost.toFixed(6)) : undefined; +} + +const PRICE_MAP_FILE = "pricing/pricing.json"; +let cachedPriceMap: PriceMap | undefined; + +/** The versioned price map shipped with the package (empty when the file is missing). */ +export function loadPriceMap(filePath?: string): PriceMap { + if (filePath) return readPriceMap(filePath); + cachedPriceMap ??= readPriceMap(path.join(getPackageRootDir(), PRICE_MAP_FILE)); + return cachedPriceMap; +} + +function readPriceMap(filePath: string): PriceMap { + try { + const parsed = JSON.parse(fs.readFileSync(filePath, "utf8")) as Partial; + return { as_of: parsed.as_of ?? "unknown", models: parsed.models ?? {} }; + } catch { + return { as_of: "unknown", models: {} }; + } +} + +/** Find a fully priced entry for the model id, trying each alias in turn. */ +export function resolveModelPrice( + model: string | undefined, + priceMap: PriceMap, +): { key: string; price: ModelPrice } | undefined { + if (!model) return undefined; + const keys = Object.keys(priceMap.models); + for (const candidate of modelPriceCandidates(model)) { + const key = candidate.includes("/") + ? keys.find((entry) => entry === candidate) + : uniqueMatch(keys, (entry) => entry.slice(entry.indexOf("/") + 1) === candidate); + if (!key) continue; + const price = priceMap.models[key]; + return isPriced(price) ? { key, price } : undefined; + } + return undefined; +} + +function isPriced(price: ModelPrice | undefined): price is ModelPrice { + return ( + !!price && + (price.cache_write_input_per_m == null || + (typeof price.cache_write_input_per_m === "number" && + Number.isFinite(price.cache_write_input_per_m) && + price.cache_write_input_per_m >= 0)) && + [price.input_per_m, price.cached_input_per_m, price.output_per_m].every( + (rate) => typeof rate === "number" && Number.isFinite(rate) && rate >= 0, + ) + ); +} + +const HARNESS_DEFAULT_MODELS: Record = { + "codex/default": "openai/gpt-5.4-mini", +}; + +const PROVIDER_ALIASES: Record = { + xai: ["spacexai", "x-ai"], + spacexai: ["xai", "x-ai"], + "x-ai": ["spacexai", "xai"], + zai: ["z-ai"], + "z-ai": ["zai"], + alibaba: ["qwen"], + qwen: ["alibaba"], +}; + +/** + * Alias forms of a configured model id, most specific first: + * `gateway/` prefixes dropped, harness defaults expanded, version dashes as + * dots (`claude-sonnet-4-6` → `claude-sonnet-4.6`), `-preview` and trailing + * date segments stripped, provider spellings swapped, and finally the + * bare model name (matched only when one provider carries it). + */ +export function modelPriceCandidates(model: string): string[] { + let id = model.trim(); + while (id.startsWith("gateway/")) id = id.slice("gateway/".length); + id = HARNESS_DEFAULT_MODELS[id] ?? id; + const slash = id.indexOf("/"); + const provider = slash >= 0 ? id.slice(0, slash) : undefined; + const name = slash >= 0 ? id.slice(slash + 1) : id; + + // Only suffixes that never distinguish one priced model from another are + // stripped; version segments (`-4-6`, `-mini`) stay so a sibling model's + // price is never borrowed. + const names = new Set(); + for (const base of [name, name.replace(/(\d)-(?=\d)/g, "$1.")]) { + names.add(base); + const undated = base.replace(/-\d{8}$/u, "").replace(/-\d{2}-\d{4}$/u, ""); + names.add(undated); + names.add(undated.replace(/-preview$/u, "")); + } + + const providers = provider ? [provider, ...(PROVIDER_ALIASES[provider] ?? [])] : []; + const candidates: string[] = []; + for (const candidateName of names) { + for (const candidateProvider of providers) + candidates.push(`${candidateProvider}/${candidateName}`); + } + for (const candidateName of names) candidates.push(candidateName); + return [...new Set(candidates)]; +} + +function uniqueMatch(items: T[], predicate: (item: T) => boolean): T | undefined { + const matches = items.filter(predicate); + return matches.length === 1 ? matches[0] : undefined; +} diff --git a/packages/evals/framework/evalSystemPrompt.ts b/packages/evals/framework/evalSystemPrompt.ts new file mode 100644 index 000000000..e923e647e --- /dev/null +++ b/packages/evals/framework/evalSystemPrompt.ts @@ -0,0 +1,3 @@ +/** Evaluations have no operator available to answer follow-up questions. */ +export const EVAL_SYSTEM_PROMPT = + "Do not ask for clarification. Make a reasonable assumption and proceed."; diff --git a/packages/evals/framework/harnesses/externalRunner.ts b/packages/evals/framework/harnesses/externalRunner.ts index 82dcfc122..46d0a18cd 100644 --- a/packages/evals/framework/harnesses/externalRunner.ts +++ b/packages/evals/framework/harnesses/externalRunner.ts @@ -1,11 +1,17 @@ import type { ProbeEvidence, TaskSpec, Trajectory } from "stagehand-v3"; +import type { HarnessTrajectory, TerminationReason } from "./trajectoryAdapter.js"; import { sanitizeErrorMessage } from "@browserbasehq/stagehand-integrations/harness"; +import type { BrowserSessionLoss } from "../../core/contracts/tool.js"; import type { EvalLogger } from "../../logger.js"; +import { EVAL_SYSTEM_PROMPT } from "../evalSystemPrompt.js"; import { datasetPromptGuidance } from "../externalHarnessPlan.js"; import type { ExternalHarnessTaskPlan } from "../externalHarnessPlan.js"; import type { StepObservation } from "../observationRecorder.js"; import type { TaskResult } from "../types.js"; import { gradeExternalTrajectory, type ExternalHarnessVerifierConfig } from "../verifierAdapter.js"; +import { emitTrajectoryTrace } from "./traceLog.js"; +import { resolveBilledCost, type BilledCost } from "../costEstimate.js"; +import { normalizeUsage, type NormalizedUsage } from "../usageNormalization.js"; export type MetricValue = { count: number; value: number }; @@ -45,10 +51,13 @@ export function parseEvalResult(raw: string): ParsedEvalResult { const markerIndex = markerMatch?.index ?? -1; const resultText = markerIndex >= 0 ? raw.slice(markerIndex + (markerMatch?.[0].length ?? 0)).trim() : raw.trim(); + // Without a marker the report may trail free-form narration ("I'll open + // the site...\n\n{...}"): a report-shaped object that ends the message is + // the agent's conclusion. One quoted mid-prose is not. const candidates = markerIndex >= 0 ? [resultText, resultText.split(/\r?\n/, 1)[0]?.trim(), extractFirstJsonObject(resultText)] - : [resultText, resultText.split(/\r?\n/, 1)[0]?.trim()]; + : [resultText, resultText.split(/\r?\n/, 1)[0]?.trim(), trailingEvalResultJson(resultText)]; for (const candidate of candidates) { if (!candidate) continue; @@ -58,6 +67,30 @@ export function parseEvalResult(raw: string): ParsedEvalResult { return { success: false, raw }; } +/** + * The answer the verifier should grade: the structured report's finalAnswer + * when the agent produced one, otherwise its last message. Remove recognized + * eval reports without discarding legitimate JSON task deliverables. + */ +export function resolveFinalAnswer( + parsed: Pick, + lastMessage: string | undefined, +): string | undefined { + if (parsed.finalAnswer !== undefined) return parsed.finalAnswer; + if (!lastMessage) return undefined; + const stripped = stripEmbeddedEvalReports(lastMessage).trim(); + return stripped || undefined; +} + +/** Remove eval report envelopes, preserving unrelated JSON content. */ +export function stripEmbeddedEvalReports(text: string): string { + let output = text; + for (const span of extractJsonObjects(text)) { + if (isEvalResultJson(span)) output = output.replace(span, ""); + } + return output.replace(/\n{3,}/gu, "\n\n"); +} + export interface ExternalHarnessPromptInput { plan: ExternalHarnessTaskPlan; toolInstructions?: string; @@ -107,6 +140,12 @@ export interface ExternalHarnessUsage { cacheCreationInputTokens?: number; reasoningOutputTokens?: number; totalTokens: number; + /** + * `false` when the SDK exposed no usage at all (cursor; codex after an + * aborted turn with no rollout to recover from). Zeros with `reported: false` + * are treated as unknown, never as a free run. + */ + reported?: boolean; } export interface ExternalHarnessSessionOutcome { @@ -126,6 +165,26 @@ export interface ExternalHarnessToolAdapterLike { captureEvidence?: () => Promise; drainStepObservations?: () => Promise; observedToolMatcher?: (name: string) => boolean; + browserSessionLoss?: () => BrowserSessionLoss | undefined; +} + +/** harnessStopReason recorded when the mounted browser died before the agent finished. */ +export const BROWSER_SESSION_LOST_STOP_REASON = "browser_session_lost"; + +/** + * Collapse a harness's normalized status + stop reason into why the run ended. + * Every SDK reports `completed | max_turns | sdk_error`; the stop reason is the + * only place aborts and browser loss are distinguishable from other errors. + */ +export function deriveTerminationReason( + outcome: Pick, "status" | "stopReason">, +): TerminationReason { + if (outcome.status === "completed") return "completed"; + if (outcome.status === "max_turns") return "step_budget"; + const stopReason = outcome.stopReason ?? ""; + if (stopReason === BROWSER_SESSION_LOST_STOP_REASON) return "browser_session_lost"; + if (/\b(aborted|interrupted)\b/iu.test(stopReason)) return "aborted"; + return "sdk_error"; } export interface ExternalHarnessTrajectoryInput { @@ -140,15 +199,32 @@ export interface ExternalHarnessTrajectoryInput { export interface RunExternalHarnessTaskInput { harness: string; + /** Execution implementation; absence on historical rows means unknown. */ + implementation?: { name: string; version: number; sdkVersion?: string }; plan: ExternalHarnessTaskPlan; + /** Configured model id, used to price the normalized token usage. */ + model?: string; logger: EvalLogger; toolAdapter?: ExternalHarnessToolAdapterLike; verifier?: ExternalHarnessVerifierConfig; resultContract: EvalResultContract; fallbackErrorMessage: string; + /** + * The step (or turn) budget the session was started with, emitted as the + * `step_budget` metric so rows can be grouped by it in Braintrust. + */ + stepBudget?: number; + /** The harness's actual budget counter; different SDKs count different units. */ + stepBudgetUnit?: "tool_calls" | "successful_tool_calls" | "model_steps" | "agent_steps" | "turns"; + configuration?: Record; /** Harness-specific result parser; defaults to the strict parseEvalResult. */ parseResult?: (raw: string) => ParsedEvalResult; - runSession: (prompt: string) => Promise>; + /** Use a native system/developer channel when the harness exposes one. */ + systemPromptMode?: "native" | "task_prefix"; + runSession: ( + prompt: string, + systemPrompt: string, + ) => Promise>; toTrajectory: (input: ExternalHarnessTrajectoryInput, taskSpec: TaskSpec) => Trajectory; } @@ -159,13 +235,19 @@ export interface RunExternalHarnessTaskInput { */ export async function runExternalHarnessTask({ harness, + implementation, plan, + model, logger, toolAdapter, verifier, resultContract, fallbackErrorMessage, + stepBudget, + stepBudgetUnit, + configuration, parseResult, + systemPromptMode = "task_prefix", runSession, toTrajectory, }: RunExternalHarnessTaskInput): Promise { @@ -174,20 +256,55 @@ export async function runExternalHarnessTask({ toolInstructions: toolAdapter?.promptInstructions, resultContract, }); - const outcome = await runSession(prompt); + const harnessConfiguration = { + ...configuration, + evalPolicyVersion: 1, + systemPromptMode, + ...(stepBudget !== undefined && { stepBudget }), + ...(stepBudgetUnit && { stepBudgetUnit }), + }; + const startedAt = performance.now(); + // CLI-only harnesses without a system-instruction channel receive the same + // eval policy as a task prefix. Native channels must not also get the prefix. + const sessionOutcome = await runSession( + systemPromptMode === "native" ? prompt : `${EVAL_SYSTEM_PROMPT}\n\n${prompt}`, + systemPromptMode === "native" ? EVAL_SYSTEM_PROMPT : "", + ); + const agentWallMs = performance.now() - startedAt; + // A run that outlived its browser has no trustworthy self-report: the agent + // was answering terminal "Browser session lost" errors, not the task. + const browserSessionLoss = toolAdapter?.browserSessionLoss?.(); + if (browserSessionLoss) { + logger.warn({ + category: "stagehand_facade", + level: 1, + message: `browser session lost before the agent finished: ${browserSessionLoss.cause}`, + }); + } + const outcome: ExternalHarnessSessionOutcome = browserSessionLoss + ? { + ...sessionOutcome, + status: "sdk_error", + stopReason: BROWSER_SESSION_LOST_STOP_REASON, + iterationError: `Browser session lost (${browserSessionLoss.cause})`, + } + : sessionOutcome; const iterationErrorMessage = stringifyError(outcome.iterationError); const rawResult = [outcome.resultText, outcome.transcriptText, iterationErrorMessage] .filter(Boolean) .join("\n\n"); const trustedResultText = outcome.resultText.trim(); const parsed = { ...(parseResult ?? parseEvalResult)(trustedResultText), raw: rawResult }; + parsed.finalAnswer = resolveFinalAnswer(parsed, trustedResultText); const sanitizedStopReason = outcome.stopReason ? sanitizeErrorMessage(outcome.stopReason) : undefined; const sanitizedIterationError = iterationErrorMessage ? sanitizeErrorMessage(iterationErrorMessage) : undefined; - const sdkErrorMessage = sanitizedStopReason ?? sanitizedIterationError; + const sdkErrorMessage = browserSessionLoss + ? sanitizedIterationError + : (sanitizedStopReason ?? sanitizedIterationError); const errorMessage = sanitizeErrorMessage( outcome.status === "sdk_error" ? (sdkErrorMessage ?? fallbackErrorMessage) @@ -199,6 +316,22 @@ export async function runExternalHarnessTask({ fallbackErrorMessage, ); const prefix = legacyHarnessFieldPrefix(harness); + const terminationReason = deriveTerminationReason(outcome); + const usage = normalizeUsage({ harness, raw: outcome.usage }); + const cost = resolveBilledCost({ harness, model, usage, reportedCostUsd: outcome.costUsd }); + if (cost.cost_source === "unavailable" && usage.convention !== "unreported") { + logger.log({ + category: "cost", + level: 1, + message: `cost unavailable for ${harness} on ${model ?? "(unknown model)"} (channel ${cost.billing_channel}); cost_usd omitted`, + }); + } + const baseMetrics: Record = { + ...buildNormalizedHarnessMetrics(outcome), + ...buildUsageCostMetrics(usage, cost), + ...(stepBudget !== undefined && { step_budget: metricValue(stepBudget) }), + agent_wall_ms: metricValue(agentWallMs), + }; const baseResult: TaskResult = { _success: outcome.status === "sdk_error" ? false : parsed.success, error: outcome.status === "sdk_error" || !parsed.success ? errorMessage : undefined, @@ -206,48 +339,193 @@ export async function runExternalHarnessTask({ finalAnswer: parsed.finalAnswer, rawResult: parsed.raw, harnessStatus: outcome.status, + harnessConfiguration, + usageConvention: usage.convention, + ...(implementation && { harnessImplementation: implementation }), ...(sanitizedStopReason && { harnessStopReason: sanitizedStopReason }), + terminationReason, + agent_wall_ms: Math.round(agentWallMs), + cost_source: cost.cost_source, + billing_channel: cost.billing_channel, + ...(cost.cost_pricing && { cost_pricing: cost.cost_pricing }), + ...(cost.cost_usd !== undefined && { cost_usd: cost.cost_usd }), // Deprecated compatibility aliases; consumers should use the normalized // harnessStatus / harnessStopReason fields for newly registered harnesses. [`${prefix}Status`]: outcome.status, ...(sanitizedStopReason && { [`${prefix}StopReason`]: sanitizedStopReason }), logs: logger.getLogs(), - metrics: buildNormalizedHarnessMetrics(outcome), + metrics: baseMetrics, }; - if (!verifier) return baseResult; + if (!verifier) { + return { ...baseResult, metrics: { ...baseMetrics, total_wall_ms: metricValue(agentWallMs) } }; + } + const isFacadeTool = toolAdapter?.observedToolMatcher; const evidenceTimeoutMs = readPositiveIntEnv("EVAL_CAPTURE_EVIDENCE_TIMEOUT_MS", 15_000); + const evidenceStartedAt = performance.now(); const finalObservation = toolAdapter?.captureEvidence ? await bestEffort(toolAdapter.captureEvidence(), evidenceTimeoutMs) : undefined; const stepObservations = toolAdapter?.drainStepObservations ? await bestEffort(toolAdapter.drainStepObservations(), evidenceTimeoutMs) : undefined; + const evidenceMs = performance.now() - evidenceStartedAt; + let trajectory: HarnessTrajectory | undefined; + const verifierStartedAt = performance.now(); const gradedResult = await gradeExternalTrajectory({ - buildTrajectory: () => - toTrajectory( - { - raw: outcome.raw, - parsed, - outcome, - ...(finalObservation && { finalObservation }), - ...(stepObservations?.length && { stepObservations }), - ...(toolAdapter?.observedToolMatcher && { - observedToolName: toolAdapter.observedToolMatcher, - }), - status: outcome.status === "completed" ? "complete" : "error", - }, - verifier.taskSpec, - ), + buildTrajectory: () => { + trajectory = withTerminationReason( + toTrajectory( + { + raw: outcome.raw, + parsed, + outcome, + ...(finalObservation && { finalObservation }), + ...(stepObservations?.length && { stepObservations }), + ...(toolAdapter?.observedToolMatcher && { + observedToolName: toolAdapter.observedToolMatcher, + }), + status: outcome.status === "completed" ? "complete" : "error", + }, + verifier.taskSpec, + ), + terminationReason, + ); + if (implementation) trajectory.harnessImplementation = implementation; + trajectory.harnessConfiguration = harnessConfiguration; + if (cost.cost_pricing) trajectory.cost_pricing = cost.cost_pricing; + // The readable step trace is derived from the normalized trajectory so + // every harness logs the same shape; a formatting bug must never fail + // the grade. + try { + emitTrajectoryTrace(logger, { + trajectory, + outcome: { ...outcome, stopReason: sanitizedStopReason }, + usage, + agentWallMs, + isFacadeTool, + report: { + summary: parsed.summary, + finalAnswer: parsed.finalAnswer, + success: parsed.success, + }, + }); + } catch (traceError) { + logger.warn({ + category: "trace", + level: 1, + message: `step trace failed: ${stringifyError(traceError)}`, + }); + } + return trajectory; + }, verifier, baseResult, errorMessage, category: harness, logger, + isFacadeTool, + }); + const verifierWallMs = performance.now() - verifierStartedAt; + const timing = buildTimingMetrics({ agentWallMs, evidenceMs, verifierWallMs }); + logger.log({ + category: "trace", + level: 1, + message: [ + "timing", + `agent=${formatSeconds(agentWallMs)}`, + `evidence=${formatSeconds(evidenceMs)}`, + `verifier=${formatSeconds(verifierWallMs)}`, + `total=${formatSeconds(timing.total_wall_ms.value)}`, + ...(cost.cost_usd !== undefined ? [`cost=$${cost.cost_usd} (${cost.cost_source})`] : []), + ].join(" · "), }); - return outcome.status === "sdk_error" - ? { ...gradedResult, _success: false, error: errorMessage } - : gradedResult; + const facadeMetrics = + trajectory && isFacadeTool ? buildFacadeToolCallMetrics(trajectory, isFacadeTool) : {}; + const gradedMetrics = (gradedResult.metrics ?? {}) as Record; + const result: TaskResult = { + ...gradedResult, + metrics: { ...gradedMetrics, ...facadeMetrics, ...timing }, + // Re-read so the trace and verifier lines logged after baseResult was + // built ship with the row. + logs: logger.getLogs(), + }; + // Execution failures remain visible in harnessStatus/terminationReason. The + // verifier can still establish a completed task from the captured evidence. + return result; +} + +function withTerminationReason( + trajectory: Trajectory, + terminationReason: TerminationReason, +): HarnessTrajectory { + return { ...trajectory, terminationReason }; +} + +/** + * How often the agent actually reached the mounted browser surface. A run that + * "passes" with zero facade calls answered from somewhere else (curl, another + * MCP server, prior knowledge), which the rubric verifier cannot see. + * + * Failed calls remain failures even when their text claims session loss. The + * run-level loss signal is runner-owned; normalized steps do not currently + * carry trusted per-call loss attribution. + */ +export function buildFacadeToolCallMetrics( + trajectory: Pick, + isFacadeTool: (name: string) => boolean, +): Record { + let calls = 0; + let failures = 0; + for (const step of trajectory.steps) { + if (!isFacadeTool(step.actionName)) continue; + calls += 1; + if (step.toolOutput?.ok !== false) continue; + failures += 1; + } + return { + facade_tool_calls: metricValue(calls), + facade_tool_call_failures: metricValue(failures), + }; +} + +/** Wall-clock split so agent speed is never confounded with verifier speed. */ +export function buildTimingMetrics(timing: { + agentWallMs: number; + evidenceMs: number; + verifierWallMs: number; +}): Record<"agent_wall_ms" | "evidence_ms" | "verifier_wall_ms" | "total_wall_ms", MetricValue> { + return { + agent_wall_ms: metricValue(timing.agentWallMs), + evidence_ms: metricValue(timing.evidenceMs), + verifier_wall_ms: metricValue(timing.verifierWallMs), + total_wall_ms: metricValue(timing.agentWallMs + timing.evidenceMs + timing.verifierWallMs), + }; +} + +export function formatSeconds(ms: number): string { + return `${(ms / 1000).toFixed(1)}s`; +} + +/** + * Convention-independent token buckets (the efficiency axis) plus the single + * billed cost column `cost_usd` — reported by the harness's channel, else + * computed at provider list price for direct-API harnesses, else absent + * rather than zero. Unreported usage gets no usage_* metrics rather than zeros. + */ +export function buildUsageCostMetrics( + usage: NormalizedUsage, + cost: BilledCost, +): Record { + return { + ...(usage.convention !== "unreported" && { + usage_input_total: metricValue(usage.input_total), + usage_input_cached: metricValue(usage.input_cached), + usage_output: metricValue(usage.output), + usage_reasoning: metricValue(usage.reasoning), + }), + ...(cost.cost_usd !== undefined && { cost_usd: metricValue(cost.cost_usd) }), + }; } /** Convert a registered harness id to its deprecated TaskResult field prefix. */ @@ -290,13 +568,25 @@ function toFiniteNumber(value: unknown): number { } function extractFirstJsonObject(value: string): string | undefined { - const start = value.indexOf("{"); - if (start < 0) return undefined; + return extractJsonObjects(value)[0]; +} + +/** Every top-level balanced `{...}` span in document order (not validated). */ +function extractJsonObjects(value: string): string[] { + const spans: string[] = []; + let start = -1; let depth = 0; let inString = false; let escaped = false; - for (let index = start; index < value.length; index += 1) { + for (let index = 0; index < value.length; index += 1) { const character = value[index]; + if (start < 0) { + if (character === "{") { + start = index; + depth = 1; + } + continue; + } if (inString) { if (escaped) escaped = false; else if (character === "\\") escaped = true; @@ -307,10 +597,34 @@ function extractFirstJsonObject(value: string): string | undefined { else if (character === "{") depth += 1; else if (character === "}") { depth -= 1; - if (depth === 0) return value.slice(start, index + 1); + if (depth === 0) { + spans.push(value.slice(start, index + 1)); + start = -1; + } } } - return undefined; + return spans; +} + +function trailingEvalResultJson(text: string): string | undefined { + const last = extractJsonObjects(text).at(-1); + return last && text.endsWith(last) && isEvalResultJson(last) ? last : undefined; +} + +function isEvalResultJson(candidate: string): boolean { + try { + const parsed: unknown = JSON.parse(candidate); + return ( + typeof parsed === "object" && + parsed !== null && + typeof (parsed as { success?: unknown }).success === "boolean" && + (typeof (parsed as { summary?: unknown }).summary === "string" || + typeof (parsed as { finalAnswer?: unknown }).finalAnswer === "string") && + Object.keys(parsed).every((key) => ["success", "summary", "finalAnswer"].includes(key)) + ); + } catch { + return false; + } } function tryParseEvalJson(candidate: string): Omit | undefined { diff --git a/packages/evals/framework/harnesses/toolSurfaceResolution.ts b/packages/evals/framework/harnesses/toolSurfaceResolution.ts index 977b12601..47cb31c14 100644 --- a/packages/evals/framework/harnesses/toolSurfaceResolution.ts +++ b/packages/evals/framework/harnesses/toolSurfaceResolution.ts @@ -11,6 +11,8 @@ function formatList(values: ToolSurface[]): string { return `${values.slice(0, -1).join(", ")}, or ${values.at(-1)}`; } +const DEFAULT_TOOL_SURFACE_PREFERENCE: ToolSurface[] = ["stagehand_facade"]; + /** Resolve the tool surface for a row on `harness`. */ export function resolveToolSurface( harness: Pick, @@ -18,7 +20,14 @@ export function resolveToolSurface( ): ToolSurface | undefined { const supported = harness.supportedToolSurfaces; if (supported.length === 0) return requested; - if (requested === undefined) return supported[0]; + // Default to the facade wherever the harness supports it: it is the surface + // every benchmark comparison is run on, and the list's first entry + // (browse_cli / playwright_code) has silently produced invalid runs when + // --tool was omitted. + if (requested === undefined) + return ( + DEFAULT_TOOL_SURFACE_PREFERENCE.find((surface) => supported.includes(surface)) ?? supported[0] + ); if (supported.includes(requested)) return requested; throw new EvalsError( `Harness "${harness.harness}" supports --tool ${formatList(supported)}; received "${requested}".`, diff --git a/packages/evals/framework/harnesses/traceLog.ts b/packages/evals/framework/harnesses/traceLog.ts new file mode 100644 index 000000000..3d5d30940 --- /dev/null +++ b/packages/evals/framework/harnesses/traceLog.ts @@ -0,0 +1,294 @@ +import type { LogLine, Trajectory, TrajectoryStep } from "stagehand-v3"; +import type { ExternalHarnessSessionOutcome } from "./externalRunner.js"; +import { formatNormalizedUsage, type NormalizedUsage } from "../usageNormalization.js"; + +/** + * Readable per-step trace shared by every external harness. It is derived + * from the normalized Trajectory, so claude_code, codex, mastra, pi, eve, + * deepagents, fx and cursor all log the exact same shape: + * + * step 3 · think · + * step 3 · run · ok · await page.goto('https://…'); return page.title() → "Recreation.gov…" + * step 4 · screenshot · ok → [image 42 KB] + * step 5 · run · ERR · await page.click('#nope') → Timeout 30000ms exceeded + * summary · + * answer · (or "answer · (none — max_turns)") + * result · completed · steps=5 · facade_calls=4 · in=12345 (cached 9000) out=678 · agent=42.0s + * + * The verifier runs after this trace, so its wall-clock lands on a separate + * `timing · agent=… evidence=… verifier=… total=…` line once grading is done. + * + * Full code and results travel in `auxiliary` so Braintrust / parseLogLine + * keep the detail expandable without polluting the message text. + */ + +export const TRACE_LOG_CATEGORY = "trace"; +export const TRACE_CLIP_CHARS = 200; +/** Cap on full-detail auxiliary payloads so a 300 KB snapshot cannot bloat a row. */ +export const TRACE_AUXILIARY_MAX_CHARS = 16_000; + +const SEPARATOR = " · "; +const ARROW = " → "; + +export interface TraceLogSink { + log(line: LogLine): void; +} + +export interface TrajectoryTraceInput { + trajectory: Pick; + outcome: Pick, "status" | "stopReason" | "usage">; + /** Convention-normalized token buckets; falls back to the raw SDK usage when absent. */ + usage?: NormalizedUsage; + /** Wall-clock of the agent session alone (ms), excluding evidence capture and grading. */ + agentWallMs?: number; + /** Optional per-step wall-clock durations (ms), indexed like `trajectory.steps`. */ + stepDurationsMs?: ReadonlyArray; + /** Which action names count as calls into the mounted browser surface. */ + isFacadeTool?: (name: string) => boolean; + /** The agent's parsed self-report (EVAL_RESULT), when available. */ + report?: { summary?: string; finalAnswer?: string; success?: boolean }; +} + +/** Build every trace line for a completed run, in emission order. */ +export function buildTrajectoryTraceLines(input: TrajectoryTraceInput): LogLine[] { + const lines: LogLine[] = []; + input.trajectory.steps.forEach((step, index) => { + lines.push(...buildStepTraceLines(step, index + 1, input.stepDurationsMs?.[index])); + }); + lines.push(...buildAnswerTraceLines(input)); + lines.push(buildResultTraceLine(input)); + return lines; +} + +/** + * What the agent said it did and concluded. A missing answer is stated + * explicitly (with the stop status) — silence here hides budget/error stops + * where the agent never produced a final message. + */ +export function buildAnswerTraceLines(input: TrajectoryTraceInput): LogLine[] { + const report = input.report ?? {}; + const lines: LogLine[] = []; + const summary = report.summary?.trim(); + if (summary) { + lines.push({ + category: TRACE_LOG_CATEGORY, + level: 1, + message: ["summary", clip(singleLine(summary))].join(SEPARATOR), + auxiliary: { summary: { value: capped(summary), type: "string" } }, + }); + } + const answer = report.finalAnswer?.trim(); + if (answer) { + lines.push({ + category: TRACE_LOG_CATEGORY, + level: 1, + message: ["answer", clip(singleLine(answer))].join(SEPARATOR), + auxiliary: { answer: { value: capped(answer), type: "string" } }, + }); + } else { + const why = input.outcome.status === "completed" ? "agent reported none" : input.outcome.status; + lines.push({ + category: TRACE_LOG_CATEGORY, + level: input.outcome.status === "completed" ? 1 : 0, + message: ["answer", `(none — ${why})`].join(SEPARATOR), + }); + } + return lines; +} + +/** Emit the trace through an EvalLogger-compatible sink. */ +export function emitTrajectoryTrace(sink: TraceLogSink, input: TrajectoryTraceInput): void { + for (const line of buildTrajectoryTraceLines(input)) sink.log(line); +} + +export function buildStepTraceLines( + step: TrajectoryStep, + ordinal: number, + durationMs?: number, +): LogLine[] { + const lines: LogLine[] = []; + const reasoning = singleLine(step.reasoning); + if (reasoning) { + lines.push({ + category: TRACE_LOG_CATEGORY, + level: 1, + message: ["step " + ordinal, "think", clip(reasoning)].join(SEPARATOR), + auxiliary: { reasoning: { value: capped(step.reasoning), type: "string" } }, + }); + } + + const tool = shortToolName(step.actionName); + const ok = step.toolOutput?.ok !== false; + const code = describeArgs(tool, step.actionArgs); + const result = ok ? describeResult(tool, step) : describeError(step); + const head = [ + "step " + ordinal, + tool, + ok ? "ok" : "ERR", + ...(durationMs !== undefined ? [formatDuration(durationMs)] : []), + ...(code ? [clip(code)] : []), + ].join(SEPARATOR); + + lines.push({ + category: TRACE_LOG_CATEGORY, + level: ok ? 1 : 0, + message: result ? head + ARROW + clip(result) : head, + auxiliary: buildStepAuxiliary(step, code), + }); + return lines; +} + +export function buildResultTraceLine(input: TrajectoryTraceInput): LogLine { + const { outcome, trajectory } = input; + const facadeCalls = input.isFacadeTool + ? trajectory.steps.filter((step) => input.isFacadeTool!(step.actionName)).length + : undefined; + const raw = outcome.usage; + const tokens = input.usage + ? formatNormalizedUsage(input.usage) + : [ + `in=${raw.inputTokens}`, + `out=${raw.outputTokens}`, + ...(raw.cachedInputTokens !== undefined ? [`cached=${raw.cachedInputTokens}`] : []), + ].join(" "); + const message = [ + "result", + outcome.status, + ...(outcome.stopReason ? [singleLine(outcome.stopReason)] : []), + `steps=${trajectory.steps.length}`, + ...(facadeCalls !== undefined ? [`facade_calls=${facadeCalls}`] : []), + tokens, + ...(input.agentWallMs !== undefined ? [`agent=${(input.agentWallMs / 1000).toFixed(1)}s`] : []), + ].join(SEPARATOR); + return { + category: TRACE_LOG_CATEGORY, + level: 1, + message: clip(message, 400), + auxiliary: { + usage: { value: JSON.stringify(raw), type: "object" }, + ...(input.usage && { + normalized_usage: { value: JSON.stringify(input.usage), type: "object" }, + }), + }, + }; +} + +/** + * Collapse harness-specific tool naming onto the surface's own tool name: + * `mcp__stagehand__run`, `stagehand.run`, `stagehand_run` all become `run`. + * Names that carry no server prefix (Bash, node, web_search) pass through. + */ +export function shortToolName(actionName: string): string { + let name = actionName.trim() || "tool"; + if (name.startsWith("mcp__")) { + const separator = name.lastIndexOf("__"); + if (separator > 4) name = name.slice(separator + 2); + } else if (name.includes(".")) { + name = name.slice(name.lastIndexOf(".") + 1); + } + if (/^(?:mcp_)?stagehand_(?:browser_)?(run|snapshot|screenshot)$/u.test(name)) { + name = name.replace(/^(?:mcp_)?stagehand_(?:browser_)?/u, ""); + } + return name || "tool"; +} + +function describeArgs(tool: string, args: Record): string { + if (!args || Object.keys(args).length === 0) return ""; + if (tool === "run") { + if (typeof args.code === "string") return singleLine(args.code); + if (args.actions !== undefined) return "actions " + (safeJson(args.actions) ?? ""); + } + if (typeof args.command === "string") return singleLine(args.command); + return safeJson(args) ?? ""; +} + +function describeResult(tool: string, step: TrajectoryStep): string { + const imageBytes = imageEvidenceBytes(step); + if (imageBytes > 0) return `[image ${formatKb(imageBytes)}]`; + const result = step.toolOutput?.result; + if (result === undefined || result === null) return ""; + if (Buffer.isBuffer(result)) return `[image ${formatKb(result.length)}]`; + if (typeof result === "string") { + if (tool === "screenshot") return "[image]"; + if (tool === "snapshot") return describeSnapshot(result); + return singleLine(result); + } + return safeJson(result) ?? String(result); +} + +function describeError(step: TrajectoryStep): string { + const error = step.toolOutput?.error; + if (error) return singleLine(error); + const result = step.toolOutput?.result; + if (typeof result === "string" && result) return singleLine(result); + return safeJson(result) ?? "error"; +} + +function describeSnapshot(text: string): string { + const lines = text.split(/\r?\n/u); + const nodes = lines.filter((line) => /^\s*\[\d+(?:-\d+)?\]/u.test(line)).length; + const first = lines.find((line) => line.trim())?.trim() ?? ""; + return nodes > 0 ? `[snapshot ${nodes} nodes] ${first}`.trim() : singleLine(text); +} + +function imageEvidenceBytes(step: TrajectoryStep): number { + let total = 0; + for (const modality of step.agentEvidence?.modalities ?? []) { + if (modality.type === "image") total += modality.bytes.length; + } + return total; +} + +function buildStepAuxiliary(step: TrajectoryStep, code: string): LogLine["auxiliary"] { + const auxiliary: NonNullable = { + tool: { value: step.actionName, type: "string" }, + }; + if (code) auxiliary.code = { value: capped(code), type: "string" }; + const result = step.toolOutput?.result; + if (result !== undefined && result !== null && !Buffer.isBuffer(result)) { + auxiliary.result = + typeof result === "string" + ? { value: capped(result), type: "string" } + : { value: capped(safeJson(result) ?? String(result)), type: "object" }; + } + if (step.toolOutput?.error) { + auxiliary.error = { value: capped(step.toolOutput.error), type: "string" }; + } + return auxiliary; +} + +function formatDuration(ms: number): string { + return ms >= 1000 ? `${(ms / 1000).toFixed(1)}s` : `${Math.round(ms)}ms`; +} + +function formatKb(bytes: number): string { + return `${Math.max(1, Math.round(bytes / 1024))} KB`; +} + +function singleLine(value: string | undefined): string { + return (value ?? "").replace(/\s+/gu, " ").trim(); +} + +function clip(value: string, max = TRACE_CLIP_CHARS): string { + return value.length > max ? value.slice(0, max - 1) + "…" : value; +} + +/** + * Capped auxiliary payload. Objects are capped as JSON text, which can leave a + * truncated string in a type:"object" entry — parseLogLine falls back to the + * raw string in that case. + */ +function capped(value: string): string { + return value.length > TRACE_AUXILIARY_MAX_CHARS + ? value.slice(0, TRACE_AUXILIARY_MAX_CHARS) + + `…[truncated ${value.length - TRACE_AUXILIARY_MAX_CHARS} chars]` + : value; +} + +function safeJson(value: unknown): string | undefined { + try { + return JSON.stringify(value); + } catch { + return undefined; + } +} diff --git a/packages/evals/framework/reasoningSummary.ts b/packages/evals/framework/reasoningSummary.ts new file mode 100644 index 000000000..e4f8c27c2 --- /dev/null +++ b/packages/evals/framework/reasoningSummary.ts @@ -0,0 +1,44 @@ +/** + * OpenAI Responses-style models only return reasoning text when a summary is + * requested explicitly (`reasoning: { summary }`); without it every harness + * reports reasoning tokens but an empty step.reasoning. The request is on by + * default and shared across harnesses so a run can be compared step for step. + */ +export const REASONING_SUMMARY_ENV = "EVAL_REASONING_SUMMARY"; + +export type ReasoningSummaryMode = "auto" | "concise" | "detailed"; + +export const DEFAULT_REASONING_SUMMARY: ReasoningSummaryMode = "detailed"; + +const MODES = new Set(["auto", "concise", "detailed"]); + +/** Requested summary mode, or undefined when `EVAL_REASONING_SUMMARY=off`. */ +export function readReasoningSummary( + env: NodeJS.ProcessEnv = process.env, +): ReasoningSummaryMode | undefined { + const raw = env[REASONING_SUMMARY_ENV]?.trim().toLowerCase(); + if (!raw) return DEFAULT_REASONING_SUMMARY; + if (raw === "off" || raw === "none" || raw === "false" || raw === "0") return undefined; + return MODES.has(raw) ? (raw as ReasoningSummaryMode) : DEFAULT_REASONING_SUMMARY; +} + +export function isOpenAiModel(model: string): boolean { + return !model.includes("/") || model.startsWith("openai/"); +} + +/** + * AI SDK `providerOptions` that make the OpenAI provider request reasoning + * summaries. Empty for other providers, whose reasoning text (Anthropic + * thinking) streams without being asked. + */ +export function openAiReasoningProviderOptions( + model: string, + env: NodeJS.ProcessEnv = process.env, +): Record> | undefined { + const summary = readReasoningSummary(env); + const name = model.replace(/^openai\//u, ""); + const supportsReasoning = + /^(?:gpt-[56]|o[134])(?:[.-]|$)/u.test(name) && !name.startsWith("gpt-5-chat"); + if (!summary || !isOpenAiModel(model) || !supportsReasoning) return undefined; + return { openai: { reasoningSummary: summary } }; +} diff --git a/packages/evals/framework/runner.ts b/packages/evals/framework/runner.ts index 6f74b1da9..9aadf6fad 100644 --- a/packages/evals/framework/runner.ts +++ b/packages/evals/framework/runner.ts @@ -287,6 +287,49 @@ export interface RunEvalsResult { }>; } +export type SummaryResult = RunEvalsResult["results"][number] & { categories?: string[] }; + +/** + * Normalize one Braintrust Eval result row for the run summary. Braintrust + * leaves `output` undefined when the task function threw (or its span failed + * after the task returned) and reports the failure in `error`; that row must + * still count as a failure so the summary, experiment link, and per-model + * table are written for the rest of the run. + */ +export function toSummaryResult(result: { + input: EvalInput; + output?: unknown; + error?: unknown; + metadata?: Record; +}): SummaryResult { + const output: SummaryResult["output"] = + typeof result.output === "boolean" + ? { _success: result.output } + : result.output !== null && typeof result.output === "object" + ? (result.output as SummaryResult["output"]) + : { + _success: false, + error: + formatProgressError(result.error) ?? + (result.output === undefined + ? "Braintrust reported no output for this task" + : String(result.output)), + }; + const categories = Array.isArray(result.metadata?.categories) + ? result.metadata.categories.filter( + (category): category is string => typeof category === "string", + ) + : undefined; + + return { + input: result.input, + output, + name: result.input.name, + score: output._success ? 1 : 0, + ...(categories && { categories }), + }; +} + function formatProgressError(error: unknown): string | undefined { if (error === undefined || error === null) return undefined; if (typeof error === "string") return error; @@ -299,6 +342,57 @@ function formatProgressError(error: unknown): string | undefined { } } +/** + * Experiment-level metadata for Braintrust/LangSmith filtering. Tool surface + * and model are always present (derived from the planned rows when not set + * globally) so cells can be grouped without opening a row; a single value + * is emitted as a scalar, several as a list. + */ +export function buildExperimentMetadata(input: { + environment: "LOCAL" | "BROWSERBASE"; + tier: "core" | "bench"; + coreToolSurface?: string; + coreStartupProfile?: string; + harness?: Harness; + modelOverride?: string; + useApi?: boolean; + testcases: Testcase[]; +}): Record { + const distinct = (pick: (tc: Testcase) => unknown): string[] => { + const values = new Set(); + for (const tc of input.testcases) { + const value = pick(tc); + if (typeof value === "string" && value) values.add(value); + } + return [...values].sort(); + }; + const scalarOrList = (values: string[]): string | string[] | undefined => + values.length === 0 ? undefined : values.length === 1 ? values[0] : values; + + const toolSurface = + input.coreToolSurface ?? scalarOrList(distinct((tc) => tc.metadata?.toolSurface)); + const startupProfile = + input.coreStartupProfile ?? scalarOrList(distinct((tc) => tc.metadata?.startupProfile)); + const model = + input.modelOverride ?? + scalarOrList(distinct((tc) => tc.metadata?.model).filter((m) => m !== "none")); + const provider = scalarOrList(distinct((tc) => tc.metadata?.provider)); + const dataset = scalarOrList(distinct((tc) => tc.metadata?.dataset)); + + return { + environment: input.environment, + tier: input.tier, + ...(toolSurface && { tool_surface: toolSurface, toolSurface }), + ...(startupProfile && { startup_profile: startupProfile, startupProfile }), + ...(input.harness && { harness: input.harness }), + ...(model && { model }), + ...(provider && { provider }), + ...(dataset && { dataset }), + task_count: input.testcases.length, + ...(input.useApi && { api: true }), + }; +} + const MAX_SPAN_PAYLOAD_BYTES = 2_000_000; export function capForSpan(value: Record): Record { @@ -423,19 +517,16 @@ export async function runEvals(options: RunEvalsOptions): Promise testcases, task: async (input: EvalInput): Promise => { // Cooperative abort: skip any testcase that hasn't started yet @@ -482,6 +573,7 @@ export async function runEvals(options: RunEvalsOptions): Promise { - const output = - typeof result.output === "boolean" ? { _success: result.output } : result.output; - const categories = Array.isArray(result.metadata?.categories) - ? result.metadata.categories.filter( - (category): category is string => typeof category === "string", - ) - : undefined; - - return { - input: result.input, - output, - name: result.input.name, - score: output._success ? 1 : 0, - ...(categories && { categories }), - }; - }); + const summaryResults = evalResult.results.map((result) => toSummaryResult(result)); const resolvedExperimentName = evalResult.summary?.experimentName ?? experimentName; const resolvedExperimentUrl = evalResult.summary?.experimentUrl; diff --git a/packages/evals/framework/stepBudget.ts b/packages/evals/framework/stepBudget.ts new file mode 100644 index 000000000..1dc6d87dc --- /dev/null +++ b/packages/evals/framework/stepBudget.ts @@ -0,0 +1,46 @@ +/** Env var honored by every external harness when its own key is unset. */ +export const SHARED_STEP_BUDGET_ENV = "AGENT_EVAL_MAX_STEPS"; + +/** + * Per-dataset execution budgets. Reaching a budget records step_budget; + * the verifier still determines task completion from the captured evidence. + */ +export const DATASET_STEP_BUDGETS: Readonly> = { + hardbenchmark: 100, +}; + +export interface ResolveStepBudgetInput { + /** Harness-specific env key (e.g. EVAL_CODEX_MAX_STEPS, EVAL_CLAUDE_CODE_MAX_TURNS). */ + harnessEnvKey: string; + dataset: string | undefined; + /** The harness's historical default, used when no env or dataset budget applies. */ + harnessDefault: number; + env?: NodeJS.ProcessEnv; +} + +/** + * Resolve the agent step budget for one run. Precedence: + * harness env key → AGENT_EVAL_MAX_STEPS → DATASET_STEP_BUDGETS[dataset] → harnessDefault. + * + * The unit is whatever the harness counts (tool steps for most, turns for + * claude_code and pi). A shared number does not equate those units; runners + * record the effective value and unit for comparison. + */ +export function resolveStepBudget({ + harnessEnvKey, + dataset, + harnessDefault, + env = process.env, +}: ResolveStepBudgetInput): number { + for (const key of [harnessEnvKey, SHARED_STEP_BUDGET_ENV]) { + const parsed = readPositiveInt(env[key]); + if (parsed !== undefined) return parsed; + } + const datasetBudget = dataset ? DATASET_STEP_BUDGETS[dataset] : undefined; + return datasetBudget ?? harnessDefault; +} + +function readPositiveInt(raw: string | undefined): number | undefined { + const parsed = Number(raw); + return Number.isSafeInteger(parsed) && parsed > 0 ? parsed : undefined; +} diff --git a/packages/evals/framework/usageNormalization.ts b/packages/evals/framework/usageNormalization.ts new file mode 100644 index 000000000..4f991238d --- /dev/null +++ b/packages/evals/framework/usageNormalization.ts @@ -0,0 +1,158 @@ +import type { ExternalHarnessUsage } from "./harnesses/externalRunner.js"; + +/** + * How a harness SDK lays out the token buckets it reports. + * + * - `openai_cached_subset`: `inputTokens` is the whole prompt; cached (and any + * cache-write) tokens are a subset of it. Reasoning tokens are a subset of + * `outputTokens`. This is the OpenAI Responses shape and also what the AI SDK + * (v6+) and LangChain normalize every provider onto. + * - `anthropic_cache_separate`: `inputTokens` excludes cache reads and cache + * writes; both are reported in their own fields. The raw Anthropic Messages + * API shape, surfaced unchanged by the Claude Agent SDK. + * - `uncached_only`: the harness re-normalizes every provider onto an + * Anthropic-style split, so `inputTokens` is always the uncached remainder + * (pi: `usage.input` = 40 alongside 127k `cacheRead` observed). + * - `unreported`: the SDK exposes no token usage at all; every bucket is 0 and + * must not be priced. + */ +export type UsageConvention = + | "openai_cached_subset" + | "anthropic_cache_separate" + | "uncached_only" + | "unreported"; + +export interface NormalizedUsage { + /** Every prompt token billed on any input rate: uncached + cached + cache writes. */ + input_total: number; + /** Prompt tokens served from a prompt cache. */ + input_cached: number; + /** Prompt tokens written into a prompt cache (0 when the SDK does not split them out). */ + input_cache_write: number; + /** Prompt tokens billed at the plain input rate. */ + input_uncached: number; + /** Completion tokens, including reasoning when `reasoning_in_output`. */ + output: number; + /** Reasoning / thinking tokens the SDK reported (0 when it does not). */ + reasoning: number; + /** Whether `reasoning` is already counted inside `output`. */ + reasoning_in_output: boolean; + convention: UsageConvention; +} + +export interface NormalizeUsageInput { + harness: string; + /** + * Model provider (e.g. "anthropic"). Accepted for forward compatibility; today + * every harness SDK reports one shape regardless of provider, so the + * convention is decided per harness. + */ + provider?: string; + raw: ExternalHarnessUsage; +} + +/** + * Convention per registered harness, decided from what each SDK actually + * reports (see the runner's usage extraction, not its docs): + * + * | harness | evidence | + * |-------------|---------------------------------------------------------------------------------------| + * | claude_code | result message `usage.input_tokens` + separate `cache_read/creation_input_tokens` | + * | codex | `turn.completed` usage: `cached_input_tokens` ⊂ `input_tokens`, reasoning ⊂ output | + * | mastra | @mastra/core 1.57 on ai@7: `inputTokens` total, `cachedInputTokens` subset | + * | eve | eve 0.29 reads AI SDK 7 `usage.inputTokens` + `inputTokenDetails.cacheReadTokens` | + * | deepagents | LangChain `usage_metadata.input_tokens` total, `input_token_details.cache_read` subset | + * | fx | `usage-v2.json`: cached input is a subset; reasoning_tokens is separate from output_tokens | + * | pi | pi-ai `usage.input` is the uncached remainder; `cacheRead`/`cacheWrite` separate | + * | cursor | SDK input excludes cache; historical CLI records explicitly report no usage | + * | cursor_sdk | SDK `totalTokens` sums input, output, cacheRead and cacheWrite; input excludes cache | + * | claude_cua | raw Messages API `usage.input_tokens` + separate `cache_read/creation_input_tokens` | + */ +const HARNESS_CONVENTIONS: Readonly> = { + claude_code: "anthropic_cache_separate", + codex: "openai_cached_subset", + mastra: "openai_cached_subset", + eve: "openai_cached_subset", + deepagents: "openai_cached_subset", + fx: "openai_cached_subset", + pi: "uncached_only", + cursor: "uncached_only", + cursor_sdk: "uncached_only", + claude_cua: "anthropic_cache_separate", + gemini_cua: "openai_cached_subset", +}; + +/** Unknown harnesses get the most common SDK shape; the metric is still labelled. */ +const DEFAULT_CONVENTION: UsageConvention = "openai_cached_subset"; + +export function usageConventionFor(harness: string): UsageConvention { + return Object.hasOwn(HARNESS_CONVENTIONS, harness) + ? HARNESS_CONVENTIONS[harness] + : DEFAULT_CONVENTION; +} + +export function normalizeUsage({ harness, raw }: NormalizeUsageInput): NormalizedUsage { + const input = nonNegative(raw.inputTokens); + const cached = nonNegative(raw.cachedInputTokens); + const cacheWrite = nonNegative(raw.cacheCreationInputTokens); + const output = nonNegative(raw.outputTokens); + const reasoning = nonNegative(raw.reasoningOutputTokens); + // Legacy runners initialize absent telemetry to zero. Without an explicit + // presence flag, zeros cannot establish a free run. Nonzero component + // buckets remain usable for legacy records; reported:true preserves an + // actual zero-valued usage event. Total alone cannot price missing buckets. + const declaredConvention = usageConventionFor(harness); + const hasUsage = + raw.reported === true || + input > 0 || + output > 0 || + (declaredConvention !== "openai_cached_subset" && (cached > 0 || cacheWrite > 0)); + const convention = raw.reported === false || !hasUsage ? "unreported" : declaredConvention; + + switch (convention) { + case "openai_cached_subset": + return { + input_total: input, + input_cached: Math.min(cached, input), + input_cache_write: Math.min(cacheWrite, Math.max(0, input - cached)), + input_uncached: Math.max(0, input - cached - cacheWrite), + output, + reasoning, + reasoning_in_output: harness !== "fx", + convention, + }; + case "anthropic_cache_separate": + case "uncached_only": + return { + input_total: input + cached + cacheWrite, + input_cached: cached, + input_cache_write: cacheWrite, + input_uncached: input, + output, + reasoning, + reasoning_in_output: true, + convention, + }; + case "unreported": + return { + input_total: 0, + input_cached: 0, + input_cache_write: 0, + input_uncached: 0, + output: 0, + reasoning: 0, + reasoning_in_output: true, + convention, + }; + } +} + +/** `in= (cached ) out=` for the trace result line. */ +export function formatNormalizedUsage(usage: NormalizedUsage): string { + if (usage.convention === "unreported") return "in=? out=? (usage unreported)"; + return `in=${usage.input_total} (cached ${usage.input_cached}) out=${usage.output}`; +} + +function nonNegative(value: number | undefined): number { + return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : 0; +} diff --git a/packages/evals/initStagehand.ts b/packages/evals/initStagehand.ts index dd7d60faa..bbc43eaa6 100644 --- a/packages/evals/initStagehand.ts +++ b/packages/evals/initStagehand.ts @@ -10,6 +10,7 @@ import { import type { EvalLogger } from "./logger.js"; import { launchRunnerProvidedBrowserbaseChrome } from "./core/targets/browserbase.js"; import { onceAsync, registerActiveRunCleanup } from "./framework/activeRunCleanup.js"; +import { EVAL_SYSTEM_PROMPT } from "./framework/evalSystemPrompt.js"; import { resolveKey } from "./tui/welcomeStatus.js"; export type InitStagehandArgs = { @@ -136,6 +137,7 @@ export async function initStagehand({ try { stagehand = await Stagehand.create({ browser, + systemPrompt: EVAL_SYSTEM_PROMPT, // selfHeal defaults off on the server; without it the heal_* benchmarks // pass while measuring nothing. selfHeal: true, diff --git a/packages/evals/pricing/pricing.json b/packages/evals/pricing/pricing.json new file mode 100644 index 000000000..58bffb4b2 --- /dev/null +++ b/packages/evals/pricing/pricing.json @@ -0,0 +1,180 @@ +{ + "as_of": "2026-08-31", + "models": { + "openai/gpt-5.4-mini": { + "input_per_m": 0.75, + "cached_input_per_m": 0.075, + "output_per_m": 4.5, + "source": "https://ai-gateway.vercel.sh/v1/models (openai/gpt-5.4-mini)" + }, + "openai/gpt-5.5": { + "input_per_m": 5, + "cached_input_per_m": 0.5, + "output_per_m": 30, + "source": "https://ai-gateway.vercel.sh/v1/models (openai/gpt-5.5)" + }, + "anthropic/claude-sonnet-4.6": { + "input_per_m": 3, + "cached_input_per_m": 0.3, + "cache_write_input_per_m": 3.75, + "output_per_m": 15, + "source": "https://ai-gateway.vercel.sh/v1/models (anthropic/claude-sonnet-4.6)" + }, + "anthropic/claude-opus-4.8": { + "input_per_m": 5, + "cached_input_per_m": 0.5, + "cache_write_input_per_m": 6.25, + "output_per_m": 25, + "source": "https://ai-gateway.vercel.sh/v1/models (anthropic/claude-opus-4.8)" + }, + "anthropic/claude-sonnet-5": { + "input_per_m": 2, + "cached_input_per_m": 0.2, + "cache_write_input_per_m": 2.5, + "output_per_m": 10, + "source": "https://ai-gateway.vercel.sh/v1/models (anthropic/claude-sonnet-5)" + }, + "anthropic/claude-opus-5": { + "input_per_m": 5, + "cached_input_per_m": 0.5, + "cache_write_input_per_m": 6.25, + "output_per_m": 25, + "source": "https://ai-gateway.vercel.sh/v1/models (anthropic/claude-opus-5)" + }, + "google/gemini-3-flash": { + "input_per_m": 0.5, + "cached_input_per_m": 0.05, + "output_per_m": 3, + "source": "https://ai-gateway.vercel.sh/v1/models (google/gemini-3-flash)" + }, + "google/gemini-3.5-flash": { + "input_per_m": 1.5, + "cached_input_per_m": 0.15, + "output_per_m": 9, + "source": "https://ai-gateway.vercel.sh/v1/models (google/gemini-3.5-flash)" + }, + "google/gemini-3.6-flash": { + "input_per_m": 0.75, + "cached_input_per_m": 0.075, + "output_per_m": 3.75, + "source": "https://ai-gateway.vercel.sh/v1/models (google/gemini-3.6-flash)" + }, + "google/gemini-3.7-flash": { + "input_per_m": 0.75, + "cached_input_per_m": 0.075, + "output_per_m": 3.75, + "source": "https://ai-gateway.vercel.sh/v1/models (google/gemini-3.7-flash)" + }, + "spacexai/grok-4.5": { + "input_per_m": 2, + "cached_input_per_m": 0.3, + "output_per_m": 6, + "source": "https://ai-gateway.vercel.sh/v1/models (spacexai/grok-4.5)" + }, + "spacexai/grok-4.6": { + "input_per_m": 2, + "cached_input_per_m": 0.5, + "output_per_m": 6, + "source": "https://ai-gateway.vercel.sh/v1/models (spacexai/grok-4.6)" + }, + "zai/glm-5.3": { + "input_per_m": 1.4, + "cached_input_per_m": 0.14, + "output_per_m": 4.4, + "source": "https://ai-gateway.vercel.sh/v1/models (zai/glm-5.3)" + }, + "zai/glm-5.3-flash": { + "input_per_m": 0.15, + "cached_input_per_m": 0.03, + "output_per_m": 0.5, + "source": "https://ai-gateway.vercel.sh/v1/models (zai/glm-5.3-flash)" + }, + "alibaba/qwen3.8-flash": { + "input_per_m": 0.16, + "cached_input_per_m": 0.016, + "cache_write_input_per_m": 0.2, + "output_per_m": 0.47, + "source": "https://ai-gateway.vercel.sh/v1/models (alibaba/qwen3.8-flash)" + }, + "deepseek/deepseek-v4-pro": { + "input_per_m": 0.66, + "cached_input_per_m": 0.022, + "output_per_m": 1.98, + "source": "https://ai-gateway.vercel.sh/v1/models (deepseek/deepseek-v4-pro)" + }, + "deepseek/deepseek-v4-flash": { + "input_per_m": 0.13, + "cached_input_per_m": 0.028, + "output_per_m": 0.26, + "source": "https://ai-gateway.vercel.sh/v1/models (deepseek/deepseek-v4-flash)" + }, + "openai/gpt-5.6-luna": { + "input_per_m": 0.2, + "cached_input_per_m": 0.02, + "output_per_m": 1.2, + "source": "https://developers.openai.com/api/docs/pricing (fetched 2026-09-02)", + "note": "public list price; matches gateway listing of 2026-08-31" + }, + "openai/gpt-5.6-sol": { + "input_per_m": 4.0, + "cached_input_per_m": 0.4, + "output_per_m": 20.0, + "source": "https://developers.openai.com/api/docs/pricing (fetched 2026-09-02)", + "note": "public list price; gateway listed out=10 on 2026-08-31, official page says 20" + }, + "openai/gpt-5.6-terra": { + "input_per_m": 2.0, + "cached_input_per_m": 0.2, + "output_per_m": 12.0, + "source": "https://developers.openai.com/api/docs/pricing (fetched 2026-09-02)", + "note": "public list price; matches gateway listing of 2026-08-31" + }, + "anthropic/claude-fable-5": { + "input_per_m": 10.0, + "cached_input_per_m": 1.0, + "output_per_m": 50.0, + "source": "https://platform.claude.com/docs/en/about-claude/pricing (fetched 2026-09-02)", + "note": "public list price; 5m cache write 12.50, 1h 20", + "cache_write_input_per_m": 12.5 + }, + "anthropic/claude-fable-5-1": { + "input_per_m": 10.0, + "cached_input_per_m": 0.25, + "output_per_m": 50.0, + "source": "https://platform.claude.com/docs/en/about-claude/pricing (fetched 2026-09-02)", + "note": "public list price; cache hits at 0.025x base input on Fable 5.1 (other models 0.1x); 5m cache write 12.50, 1h 20", + "cache_write_input_per_m": 12.5 + }, + "google/gemini-3.8-flash": { + "input_per_m": 0.75, + "cached_input_per_m": 0.075, + "output_per_m": 3.75, + "source": "https://ai.google.dev/gemini-api/docs/pricing (Gemini 3.8 Flash, promotional rate through 2026-12-31; output includes thinking tokens)" + }, + "zai/glm-5.3-promo-50": { + "input_per_m": 0.7, + "output_per_m": 2.2, + "source": "https://ai-gateway.vercel.sh/v1/models (zai/glm-5.3-promo-50)", + "cached_input_per_m": 0.13 + }, + "alibaba/qwen3.8-max-0902": { + "input_per_m": 2.0, + "output_per_m": 6.0, + "source": "https://ai-gateway.vercel.sh/v1/models (alibaba/qwen3.8-max-0902)", + "cached_input_per_m": 0.25 + }, + "meta/muse-spark-1.3": { + "input_per_m": 1.25, + "output_per_m": 4.25, + "source": "https://ai-gateway.vercel.sh/v1/models (meta/muse-spark-1.3)", + "cached_input_per_m": 0.15 + }, + "openai/gpt-6-astra": { + "input_per_m": null, + "cached_input_per_m": null, + "output_per_m": null, + "source": "needs owner input", + "note": "Astra context-length tiers require per-request accounting; aggregate run usage cannot select the correct rate. Observed base rates were input=10 cached=1 output=50 USD/M. Estimates stay unavailable until tier-aware accounting is implemented." + } + } +} diff --git a/packages/evals/scripts/update-pricing.ts b/packages/evals/scripts/update-pricing.ts new file mode 100644 index 000000000..a173ef1e4 --- /dev/null +++ b/packages/evals/scripts/update-pricing.ts @@ -0,0 +1,193 @@ +/** + * Refresh the model entries already present in the versioned price map into pricing/pricing.json. + * + * Prereqs: network access. No API key: the Vercel AI Gateway model list is + * public; OpenRouter's public models endpoint is the fallback. + * Args: none. + * Env: EVAL_PRICING_SOURCE=gateway|openrouter to force one source. + * Example: pnpm exec tsx packages/evals/scripts/update-pricing.ts + * + * Entries already marked unpriced remain at null prices and + * "source": "needs owner input" so cost estimation reports them as unpriced + * instead of $0 — even when a public catalog happens to list a price for the + * id (the observed value is kept in `note` for the owner to confirm). + */ +import fs from "node:fs"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import { EvalsError } from "../errors.js"; +import { getPackageRootDir } from "../runtimePaths.js"; +import type { ModelPrice, PriceMap } from "../framework/costEstimate.js"; + +const GATEWAY_MODELS_URL = "https://ai-gateway.vercel.sh/v1/models"; +const OPENROUTER_MODELS_URL = "https://openrouter.ai/api/v1/models"; + +/** OpenRouter spells a few creators differently from the gateway. */ +const OPENROUTER_ID_ALIASES: Record = { + "spacexai/": "x-ai/", + "zai/": "z-ai/", + "alibaba/": "qwen/", +}; + +type FetchedPrice = Omit & { source: string }; + +async function fetchCatalog( + url: string, + provider: string, +): Promise<{ data?: Array> }> { + let response: Response; + try { + response = await fetch(url); + } catch (cause) { + throw new EvalsError(`${provider} model catalog request failed.`, { cause }); + } + if (!response.ok) throw new EvalsError(`${provider} models: HTTP ${response.status}`); + try { + const body: unknown = await response.json(); + if (!body || typeof body !== "object" || !Array.isArray((body as { data?: unknown }).data)) { + throw new EvalsError(`${provider} model catalog has no model list.`); + } + return body as { data: Array> }; + } catch (cause) { + if (cause instanceof EvalsError) throw cause; + throw new EvalsError(`${provider} model catalog is not valid JSON.`, { cause }); + } +} + +async function fetchGatewayPrices(): Promise> { + const body = await fetchCatalog(GATEWAY_MODELS_URL, "gateway"); + const prices = new Map(); + for (const model of Array.isArray(body.data) ? body.data : []) { + if (!model || typeof model !== "object") continue; + const pricing = model.pricing as Record | undefined; + const id = typeof model.id === "string" ? model.id : undefined; + if (!id || !pricing) continue; + const input = perMillion(pricing.input); + const output = perMillion(pricing.output); + if (input === undefined || output === undefined) continue; + prices.set(id, { + input_per_m: input, + cached_input_per_m: perMillion(pricing.input_cache_read) ?? input, + ...(perMillion(pricing.input_cache_write) !== undefined && { + cache_write_input_per_m: perMillion(pricing.input_cache_write), + }), + output_per_m: output, + source: `${GATEWAY_MODELS_URL} (${id})`, + }); + } + if (prices.size === 0) throw new EvalsError("Model catalog contained no usable prices."); + return prices; +} + +async function fetchOpenRouterPrices(): Promise> { + const body = await fetchCatalog(OPENROUTER_MODELS_URL, "openrouter"); + const prices = new Map(); + for (const model of Array.isArray(body.data) ? body.data : []) { + if (!model || typeof model !== "object") continue; + const pricing = model.pricing as Record | undefined; + const id = typeof model.id === "string" ? model.id : undefined; + if (!id || !pricing) continue; + const input = perMillion(pricing.prompt); + const output = perMillion(pricing.completion); + if (input === undefined || output === undefined) continue; + prices.set(toGatewayId(id), { + input_per_m: input, + cached_input_per_m: perMillion(pricing.input_cache_read) ?? input, + ...(perMillion(pricing.input_cache_write) !== undefined && { + cache_write_input_per_m: perMillion(pricing.input_cache_write), + }), + output_per_m: output, + source: `${OPENROUTER_MODELS_URL} (${id})`, + }); + } + if (prices.size === 0) throw new EvalsError("Model catalog contained no usable prices."); + return prices; +} + +function toGatewayId(openRouterId: string): string { + for (const [gateway, openRouter] of Object.entries(OPENROUTER_ID_ALIASES)) { + if (openRouterId.startsWith(openRouter)) return gateway + openRouterId.slice(openRouter.length); + } + return openRouterId; +} + +/** Catalogs quote USD per token as strings; the price map stores USD per million. */ +function perMillion(value: unknown): number | undefined { + if (typeof value !== "number" && (typeof value !== "string" || !value.trim())) return undefined; + const perToken = Number(value); + if (!Number.isFinite(perToken) || perToken < 0) return undefined; + return Number((perToken * 1_000_000).toPrecision(10)); +} + +async function loadPrices(): Promise<{ prices: Map; source: string }> { + const forced = process.env.EVAL_PRICING_SOURCE; + if (forced !== "openrouter") { + try { + return { prices: await fetchGatewayPrices(), source: "gateway" }; + } catch (error) { + if (forced === "gateway") throw error; + console.warn("gateway model catalog unavailable; falling back to OpenRouter"); + } + } + return { prices: await fetchOpenRouterPrices(), source: "openrouter" }; +} + +export async function updatePricing( + target = path.join(getPackageRootDir(), "pricing", "pricing.json"), +): Promise { + const existing = JSON.parse(fs.readFileSync(target, "utf8")) as PriceMap; + const pricedModels = Object.keys(existing.models).filter( + (id) => existing.models[id].input_per_m !== null, + ); + const ownerInputModels = Object.keys(existing.models).filter( + (id) => existing.models[id].input_per_m === null, + ); + const { prices, source } = await loadPrices(); + if (pricedModels.length && !pricedModels.some((id) => prices.has(id))) { + throw new EvalsError( + "Model catalog contained no prices for currently priced models; file unchanged.", + ); + } + const models: PriceMap["models"] = {}; + const missing: string[] = []; + for (const id of pricedModels) { + const price = prices.get(id); + if (price) models[id] = price; + else { + missing.push(id); + models[id] = { + input_per_m: null, + cached_input_per_m: null, + output_per_m: null, + source: `not listed by ${source} on ${today()}; needs owner input`, + }; + } + } + for (const id of ownerInputModels) { + const observed = prices.get(id); + models[id] = { + input_per_m: null, + cached_input_per_m: null, + output_per_m: null, + source: "needs owner input", + ...(observed && { + note: `${source} lists in=${observed.input_per_m} cached=${observed.cached_input_per_m}${observed.cache_write_input_per_m !== undefined ? ` cache_write=${observed.cache_write_input_per_m}` : ""} out=${observed.output_per_m} USD/M on ${today()}; confirm before enabling this price`, + }), + }; + } + const priceMap: PriceMap = { as_of: today(), models }; + fs.mkdirSync(path.dirname(target), { recursive: true }); + fs.writeFileSync(target, JSON.stringify(priceMap, null, 2) + "\n"); + console.log( + `wrote ${target}: ${pricedModels.length - missing.length} priced from ${source}, ${ownerInputModels.length} awaiting owner input` + + (missing.length ? `, ${missing.length} not listed: ${missing.join(", ")}` : ""), + ); +} + +function today(): string { + return new Date().toISOString().slice(0, 10); +} + +if (process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) { + await updatePricing(); +} diff --git a/packages/evals/tests/framework/benchHarness.test.ts b/packages/evals/tests/framework/benchHarness.test.ts index 7c0b2e968..28ab356f2 100644 --- a/packages/evals/tests/framework/benchHarness.test.ts +++ b/packages/evals/tests/framework/benchHarness.test.ts @@ -227,6 +227,11 @@ describe("bench harness registry", () => { it("defines the shared external lifecycle and cleans up when the agent throws", async () => { let cleanupCalled = false; const adapter = { + browserSession: { + provider: "browserbase" as const, + sessionId: "session-a", + sessionUrl: "https://www.browserbase.com/sessions/session-a", + }, cleanup: async (): Promise => { cleanupCalled = true; }, @@ -289,7 +294,14 @@ describe("bench harness registry", () => { expect(harness.supportsApi).toBe(false); await expect( harness.execute?.({ task, input, row, logger: new EvalLogger(false) }), - ).rejects.toThrow("agent failed"); + ).resolves.toMatchObject({ + _success: false, + error: "agent failed", + harnessStatus: "sdk_error", + browserProvider: "browserbase", + browserbaseSessionId: "session-a", + sessionUrl: "https://www.browserbase.com/sessions/session-a", + }); expect(preparedInput).toMatchObject({ toolSurface: "browse_cli", startupProfile: "tool_create_browserbase", @@ -357,6 +369,140 @@ describe("bench harness registry", () => { } }); + it("logs the browser session as the first task log line and stamps the result", async () => { + const close = vi.spyOn(V3.prototype, "close").mockResolvedValue(undefined); + const logger = new EvalLogger(false); + let agentSawSessionLine = false; + const harness = defineExternalHarness({ + harness: "session_first_external", + supportedToolSurfaces: ["stagehand_facade"], + defaultModels: ["openai/x" as AvailableModel], + prepareToolAdapter: async (input) => { + input.logger.log({ category: "setup", message: "bridge started", level: 2 }); + return { + browserSession: { + provider: "browserbase" as const, + sessionId: "sess-1", + sessionUrl: "https://www.browserbase.com/sessions/sess-1", + }, + cleanup: async () => {}, + }; + }, + runAgent: async (input) => { + agentSawSessionLine = input.logger + .getLogs() + .some( + (line) => + line.message === "Browserbase session: https://www.browserbase.com/sessions/sess-1", + ); + input.logger.log({ category: "agent", message: "step 1 · run · ok", level: 1 }); + return { _success: true, logs: input.logger.getLogs() }; + }, + }); + const input: EvalInput = { + name: "agent/webvoyager", + modelName: "openai/x" as AvailableModel, + params: { id: "wv-1", web: "https://example.com", ques: "Find it" }, + }; + const task: DiscoveredTask = { + name: input.name, + tier: "bench", + primaryCategory: "agent", + categories: ["agent"], + tags: [], + filePath: "/tmp/fake.ts", + isLegacy: false, + }; + const row: BenchMatrixRow = { + harness: "session_first_external", + task: input.name, + category: "agent", + taskKind: "agent", + model: input.modelName, + environment: "BROWSERBASE", + useApi: false, + toolSurface: "stagehand_facade", + startupProfile: "tool_create_browserbase", + trial: 1, + config: { + harness: "session_first_external", + model: input.modelName, + environment: "BROWSERBASE", + useApi: false, + toolSurface: "stagehand_facade", + startupProfile: "tool_create_browserbase", + }, + }; + + try { + const result = await harness.execute!({ task, input, row, logger }); + expect(agentSawSessionLine).toBe(true); + // Level-2 setup chatter is filtered out, so the session pointer heads the row logs. + expect((result.logs ?? []).map((line) => line.message)).toEqual([ + "Browserbase session: https://www.browserbase.com/sessions/sess-1", + "step 1 · run · ok", + ]); + expect(result.logs?.[0]).toMatchObject({ category: "session", level: 0 }); + expect(result).toMatchObject({ + sessionUrl: "https://www.browserbase.com/sessions/sess-1", + browserbaseSessionId: "sess-1", + browserProvider: "browserbase", + }); + } finally { + close.mockRestore(); + } + }); + + it("logs a bare provider line when the adapter reports no session", async () => { + const close = vi.spyOn(V3.prototype, "close").mockResolvedValue(undefined); + const logger = new EvalLogger(false); + const harness = defineExternalHarness({ + harness: "session_fallback_external", + supportedToolSurfaces: ["browse_cli"], + defaultModels: ["openai/x" as AvailableModel], + prepareToolAdapter: async () => ({ cleanup: async () => {} }), + runAgent: async () => ({ _success: true }), + }); + const input: EvalInput = { + name: "agent/webvoyager", + modelName: "openai/x" as AvailableModel, + params: { id: "wv-1", web: "https://example.com", ques: "Find it" }, + }; + const task: DiscoveredTask = { + name: input.name, + tier: "bench", + primaryCategory: "agent", + categories: ["agent"], + tags: [], + filePath: "/tmp/fake.ts", + isLegacy: false, + }; + const row: BenchMatrixRow = { + harness: "session_fallback_external", + task: input.name, + category: "agent", + taskKind: "agent", + model: input.modelName, + environment: "LOCAL", + useApi: false, + trial: 1, + config: { + harness: "session_fallback_external", + model: input.modelName, + environment: "LOCAL", + useApi: false, + }, + }; + try { + const result = await harness.execute!({ task, input, row, logger }); + expect(logger.getLogs().map((line) => line.message)).toEqual(["Browser: local"]); + expect(result.browserProvider).toBe("local"); + expect(result.sessionUrl).toBeUndefined(); + } finally { + close.mockRestore(); + } + }); + it("rejects mismatched external harness config before preparing an adapter", async () => { let prepareCalled = false; const harness = defineExternalHarness({ diff --git a/packages/evals/tests/framework/benchPlanner.test.ts b/packages/evals/tests/framework/benchPlanner.test.ts index 33b9b022f..b3f5f34ea 100644 --- a/packages/evals/tests/framework/benchPlanner.test.ts +++ b/packages/evals/tests/framework/benchPlanner.test.ts @@ -212,12 +212,12 @@ describe("benchPlanner", () => { expect(testcases[0].input.isCUA).toBeUndefined(); expect(testcases[0].tags).toContain("harness/claude_code"); expect(testcases[0].metadata.harness).toBe("claude_code"); - expect(testcases[0].metadata.toolSurface).toBe("browse_cli"); + expect(testcases[0].metadata.toolSurface).toBe("stagehand_facade"); expect(testcases[0].metadata.startupProfile).toBe("tool_launch_local"); expect(testcases[0].metadata.agentMode).toBeUndefined(); }); - it("keeps codex as a harness-level matrix with browse_cli metadata", async () => { + it("keeps codex as a harness-level matrix with facade metadata", async () => { const testcases = await withEnvOverrides( { EVAL_MAX_K: "1", @@ -237,9 +237,9 @@ describe("benchPlanner", () => { expect(testcases[0].input.isCUA).toBeUndefined(); expect(testcases[0].tags).toContain("harness/codex"); expect(testcases[0].metadata.harness).toBe("codex"); - expect(testcases[0].metadata.toolSurface).toBe("browse_cli"); + expect(testcases[0].metadata.toolSurface).toBe("stagehand_facade"); expect(testcases[0].metadata.startupProfile).toBe("tool_launch_local"); - expect(testcases[0].metadata.toolCommand).toBe("browse"); + expect(testcases[0].metadata.toolCommand).toBeUndefined(); expect(testcases[0].metadata.agentMode).toBeUndefined(); }); diff --git a/packages/evals/tests/framework/browserSession.test.ts b/packages/evals/tests/framework/browserSession.test.ts new file mode 100644 index 000000000..5cbfbee81 --- /dev/null +++ b/packages/evals/tests/framework/browserSession.test.ts @@ -0,0 +1,100 @@ +import { describe, expect, it } from "vitest"; +import { + browserSessionFromMetadata, + buildBrowserSessionLogLines, + withBrowserSession, +} from "../../framework/browserSession.js"; + +describe("browser session info", () => { + it("reads runner-provided target metadata", () => { + expect( + browserSessionFromMetadata( + { + browserbaseSessionId: "abc-123", + browserbaseSessionUrl: "https://www.browserbase.com/sessions/abc-123", + browserbaseDebugUrl: "https://debug.example/abc-123", + }, + "BROWSERBASE", + ), + ).toEqual({ + provider: "browserbase", + sessionId: "abc-123", + sessionUrl: "https://www.browserbase.com/sessions/abc-123", + debugUrl: "https://debug.example/abc-123", + }); + }); + + it("derives the missing half of id/url from the other", () => { + expect(browserSessionFromMetadata({ browserbaseSessionId: "abc" }, "BROWSERBASE")).toEqual({ + provider: "browserbase", + sessionId: "abc", + sessionUrl: "https://www.browserbase.com/sessions/abc", + }); + expect( + browserSessionFromMetadata( + { browserbaseSessionUrl: "https://www.browserbase.com/sessions/xyz?tab=logs" }, + "BROWSERBASE", + ), + ).toMatchObject({ sessionId: "xyz" }); + }); + + it("encodes a session identifier as one URL path segment", () => { + expect( + browserSessionFromMetadata({ browserbaseSessionId: "id/with?#spaces " }, "BROWSERBASE"), + ).toMatchObject({ + sessionUrl: "https://www.browserbase.com/sessions/id%2Fwith%3F%23spaces%20", + }); + }); + + it("falls back to the bare provider", () => { + expect(browserSessionFromMetadata({ browserbaseSessionId: "ignored" }, "LOCAL")).toEqual({ + provider: "local", + }); + expect(browserSessionFromMetadata(undefined, "BROWSERBASE")).toEqual({ + provider: "browserbase", + }); + }); + + it("formats level-0 session lines", () => { + expect(buildBrowserSessionLogLines({ provider: "local" })).toEqual([ + { + category: "session", + level: 0, + message: "Browser: local", + auxiliary: { provider: { value: "local", type: "string" } }, + }, + ]); + expect( + buildBrowserSessionLogLines({ provider: "browserbase" }).map((line) => line.message), + ).toEqual(["Browser: browserbase (session id not reported by this surface)"]); + expect( + buildBrowserSessionLogLines({ + provider: "browserbase", + sessionId: "abc", + sessionUrl: "https://www.browserbase.com/sessions/abc", + debugUrl: "https://debug.example/abc", + }).map((line) => line.message), + ).toEqual([ + "Browserbase session: https://www.browserbase.com/sessions/abc", + "Browserbase debugger: https://debug.example/abc", + ]); + }); + + it("stamps the task result without clobbering harness-reported urls", () => { + expect( + withBrowserSession( + { _success: true, sessionUrl: "https://reported.example" }, + { + provider: "browserbase", + sessionId: "abc", + sessionUrl: "https://www.browserbase.com/sessions/abc", + }, + ), + ).toEqual({ + _success: true, + browserProvider: "browserbase", + browserbaseSessionId: "abc", + sessionUrl: "https://reported.example", + }); + }); +}); diff --git a/packages/evals/tests/framework/claudeCodeToolAdapter.test.ts b/packages/evals/tests/framework/claudeCodeToolAdapter.test.ts index 5acd0d939..c8fe7f7dc 100644 --- a/packages/evals/tests/framework/claudeCodeToolAdapter.test.ts +++ b/packages/evals/tests/framework/claudeCodeToolAdapter.test.ts @@ -22,8 +22,8 @@ describe("claude code tool adapter resolution", () => { delete process.env.EVAL_CLAUDE_CODE_ALLOW_UNSANDBOXED_LOCAL; }); - it("defaults Claude Code to browse_cli", () => { - expect(resolveToolSurface(claudeCodeHarness)).toBe("browse_cli"); + it("defaults Claude Code to the shared facade", () => { + expect(resolveToolSurface(claudeCodeHarness)).toBe("stagehand_facade"); }); it("defaults browse_cli startup by environment", () => { @@ -68,7 +68,7 @@ describe("claude code tool adapter resolution", () => { }); it("supports browse_cli and the code surfaces on Codex", () => { - expect(resolveToolSurface(codexHarness)).toBe("browse_cli"); + expect(resolveToolSurface(codexHarness)).toBe("stagehand_facade"); expect(resolveToolSurface(codexHarness, "browse_cli")).toBe("browse_cli"); expect(resolveToolSurface(codexHarness, "stagehand_code")).toBe("stagehand_code"); expect(resolveToolSurface(codexHarness, "playwright_code")).toBe("playwright_code"); diff --git a/packages/evals/tests/framework/costEstimate.test.ts b/packages/evals/tests/framework/costEstimate.test.ts new file mode 100644 index 000000000..be53e14a8 --- /dev/null +++ b/packages/evals/tests/framework/costEstimate.test.ts @@ -0,0 +1,471 @@ +import { describe, expect, it } from "vitest"; +import { + computeListCost, + loadPriceMap, + modelPriceCandidates, + providerOf, + resolveBilledCost, + resolveModelPrice, + type PriceMap, +} from "../../framework/costEstimate.js"; +import { normalizeUsage } from "../../framework/usageNormalization.js"; + +const priceMap: PriceMap = { + as_of: "2026-08-31", + models: { + "openai/gpt-5.4-mini": { + input_per_m: 1, + cached_input_per_m: 0.1, + output_per_m: 4, + source: "test", + }, + "anthropic/claude-sonnet-4.6": { + input_per_m: 3, + cached_input_per_m: 0.3, + cache_write_input_per_m: 3.75, + output_per_m: 15, + source: "test", + }, + "spacexai/grok-4.5": { + input_per_m: 2, + cached_input_per_m: 0.3, + output_per_m: 6, + source: "test", + }, + "google/gemini-3-flash": { + input_per_m: 0.5, + cached_input_per_m: 0.05, + output_per_m: 3, + source: "test", + }, + "anthropic/claude-fable-5": { + input_per_m: null, + cached_input_per_m: null, + output_per_m: null, + source: "needs owner input", + }, + }, +}; + +const compute = (usage: ReturnType, model: string) => + computeListCost(usage, model, priceMap); + +describe("computeListCost", () => { + it("keeps Astra estimates unavailable until per-request context tiers are represented", () => { + const usage = normalizeUsage({ + harness: "codex", + raw: { + inputTokens: 300_000, + outputTokens: 1000, + totalTokens: 301_000, + }, + }); + expect(computeListCost(usage, "openai/gpt-6-astra", loadPriceMap())).toBeUndefined(); + }); + + it("prices the OpenAI subset convention: uncached at input, cached at cache rate, reasoning inside output", () => { + const usage = normalizeUsage({ + harness: "codex", + raw: { + inputTokens: 1_000_000, + cachedInputTokens: 600_000, + outputTokens: 100_000, + reasoningOutputTokens: 40_000, + totalTokens: 1_100_000, + }, + }); + // 400k·1 + 600k·0.1 + 100k·4 = 0.4 + 0.06 + 0.4 + expect(compute(usage, "openai/gpt-5.4-mini")).toBe(0.86); + }); + + it("prices the Anthropic separate convention with cache writes at the write rate", () => { + const usage = normalizeUsage({ + harness: "claude_code", + raw: { + inputTokens: 100_000, + cachedInputTokens: 1_000_000, + cacheCreationInputTokens: 200_000, + outputTokens: 50_000, + totalTokens: 1_350_000, + }, + }); + // 100k·3 + 1M·0.3 + 200k·3.75 + 50k·15 = 0.3 + 0.3 + 0.75 + 0.75 + expect(compute(usage, "anthropic/claude-sonnet-4-6")).toBe(2.1); + }); + + it("prices pi's uncached-only input plus its separate cache buckets", () => { + const usage = normalizeUsage({ + harness: "pi", + raw: { + inputTokens: 40, + cachedInputTokens: 1_000_000, + cacheCreationInputTokens: 0, + outputTokens: 0, + totalTokens: 1_000_040, + }, + }); + expect(compute(usage, "openai/gpt-5.4-mini")).toBeCloseTo(0.10004, 6); + }); + + it("bills reasoning at the output rate only when it is reported outside output", () => { + const base = normalizeUsage({ + harness: "codex", + raw: { + inputTokens: 0, + outputTokens: 1_000_000, + reasoningOutputTokens: 500_000, + totalTokens: 0, + }, + }); + expect(compute(base, "openai/gpt-5.4-mini")).toBe(4); + expect(compute({ ...base, reasoning_in_output: false }, "openai/gpt-5.4-mini")).toBe(6); + }); + + it("rejects malformed or negative rates, including optional cache writes, and overflow", () => { + const usage = normalizeUsage({ + harness: "pi", + raw: { inputTokens: 1, cacheCreationInputTokens: 1, outputTokens: 1, totalTokens: 3 }, + }); + for (const key of [ + "input_per_m", + "cached_input_per_m", + "cache_write_input_per_m", + "output_per_m", + ] as const) { + for (const rate of [-1, NaN, Infinity, "2"]) { + const malformed = { + ...priceMap, + models: { + "openai/gpt-5.4-mini": { ...priceMap.models["openai/gpt-5.4-mini"], [key]: rate }, + }, + } as PriceMap; + expect( + computeListCost(usage, "openai/gpt-5.4-mini", malformed), + `${key}=${rate}`, + ).toBeUndefined(); + expect( + resolveBilledCost({ + harness: "pi", + model: "openai/gpt-5.4-mini", + usage, + priceMap: malformed, + }).cost_source, + ).toBe("unavailable"); + } + } + const overflow = { ...usage, input_uncached: Number.MAX_VALUE }; + expect(computeListCost(overflow, "anthropic/claude-sonnet-4.6", priceMap)).toBeUndefined(); + }); + + it("returns nothing for null-price entries, unknown models and unreported usage", () => { + const usage = normalizeUsage({ + harness: "claude_code", + raw: { inputTokens: 10, outputTokens: 10, totalTokens: 20 }, + }); + expect(compute(usage, "anthropic/claude-fable-5")).toBeUndefined(); + expect(compute(usage, "example/unknown-model")).toBeUndefined(); + expect(computeListCost(usage, undefined, priceMap)).toBeUndefined(); + const unreported = normalizeUsage({ + harness: "cursor", + raw: { inputTokens: 0, outputTokens: 0, totalTokens: 0, reported: false }, + }); + expect(compute(unreported, "openai/gpt-5.4-mini")).toBeUndefined(); + }); +}); + +describe("resolveBilledCost", () => { + it.each([ + ["claude_cua", "anthropic/claude-sonnet-4.6", "anthropic_api"], + ["gemini_cua", "google/gemini-fixture", "google_api"], + ])( + "estimates native %s provider usage with dated price provenance", + (harness, model, channel) => { + const nativeMap: PriceMap = { + as_of: "2026-08-31", + models: { + [model]: { + input_per_m: 1, + cached_input_per_m: 0.1, + output_per_m: 4, + source: "fixture price source", + }, + }, + }; + const nativeUsage = normalizeUsage({ + harness, + raw: { inputTokens: 100, outputTokens: 10, totalTokens: 110, reported: true }, + }); + expect( + resolveBilledCost({ harness, model, usage: nativeUsage, priceMap: nativeMap }), + ).toMatchObject({ + cost_source: "computed", + billing_channel: channel, + cost_pricing: { as_of: "2026-08-31", model, source: "fixture price source" }, + }); + const missing = normalizeUsage({ + harness, + raw: { inputTokens: 0, outputTokens: 0, totalTokens: 0, reported: false }, + }); + expect(resolveBilledCost({ harness, model, usage: missing, priceMap: nativeMap })).toEqual({ + cost_source: "unavailable", + billing_channel: channel, + }); + }, + ); + const usage = normalizeUsage({ + harness: "codex", + raw: { inputTokens: 1_000_000, outputTokens: 0, totalTokens: 1_000_000 }, + }); + + it("takes the harness-reported dollars first, naming the channel", () => { + expect( + resolveBilledCost({ + harness: "claude_code", + model: "anthropic/claude-sonnet-4-6", + usage, + reportedCostUsd: 4.2, + priceMap, + }), + ).toEqual({ cost_usd: 4.2, cost_source: "reported", billing_channel: "anthropic_api" }); + expect( + resolveBilledCost({ + harness: "eve", + model: "zai/glm-5.3", + usage, + reportedCostUsd: 0.01, + priceMap, + }).billing_channel, + ).toBe("ai_gateway"); + expect( + resolveBilledCost({ + harness: "pi", + model: "openai/gpt-5.4-mini", + usage, + reportedCostUsd: 0.5, + priceMap, + }).billing_channel, + ).toBe("pi_catalog"); + expect( + resolveBilledCost({ + harness: "fx", + model: "zai/glm-5.3", + usage, + reportedCostUsd: 0.02, + priceMap, + }).billing_channel, + ).toBe("fx_gateway"); + // A reported figure wins even for a priced model on a direct-API harness. + expect( + resolveBilledCost({ + harness: "pi", + model: "openai/gpt-5.4-mini", + usage, + reportedCostUsd: 1.3, + priceMap, + }), + ).toMatchObject({ cost_usd: 1.3, cost_source: "reported" }); + }); + + it("computes direct-provider harnesses at list price when nothing was reported", () => { + for (const harness of ["codex", "mastra", "deepagents", "eve", "pi"]) { + expect( + resolveBilledCost({ harness, model: "openai/gpt-5.4-mini", usage, priceMap }), + harness, + ).toEqual({ + cost_usd: 1, + cost_source: "computed", + billing_channel: "openai_api", + cost_pricing: { as_of: "2026-08-31", model: "openai/gpt-5.4-mini", source: "test" }, + }); + } + expect( + resolveBilledCost({ harness: "codex", model: "codex/default", usage, priceMap }) + .billing_channel, + ).toBe("openai_api"); + expect( + resolveBilledCost({ harness: "mastra", model: "xai/grok-4.5", usage, priceMap }), + ).toMatchObject({ + cost_usd: 2, + cost_source: "computed", + billing_channel: "xai_api", + }); + expect( + resolveBilledCost({ + harness: "deepagents", + model: "anthropic/claude-sonnet-4-6", + usage, + priceMap, + }).billing_channel, + ).toBe("anthropic_api"); + }); + + it("is unavailable, never zero, for subscription cells, unpriced models and unreported usage", () => { + expect( + resolveBilledCost({ harness: "cursor", model: "openai/gpt-5.4-mini", usage, priceMap }), + ).toEqual({ cost_source: "unavailable", billing_channel: "subscription" }); + expect( + resolveBilledCost({ + harness: "claude_code", + model: "anthropic/claude-sonnet-4-6", + usage, + priceMap, + }), + ).toEqual({ cost_source: "unavailable", billing_channel: "subscription" }); + expect(resolveBilledCost({ harness: "fx", model: "zai/glm-5.3", usage, priceMap })).toEqual({ + cost_source: "unavailable", + billing_channel: "fx_gateway", + }); + expect( + resolveBilledCost({ harness: "codex", model: "openai/gpt-5.6-luna", usage, priceMap }), + ).toEqual({ cost_source: "unavailable", billing_channel: "openai_api" }); + const unreported = normalizeUsage({ + harness: "codex", + raw: { inputTokens: 0, outputTokens: 0, totalTokens: 0, reported: false }, + }); + expect( + resolveBilledCost({ + harness: "codex", + model: "openai/gpt-5.4-mini", + usage: unreported, + priceMap, + }), + ).toEqual({ cost_source: "unavailable", billing_channel: "openai_api" }); + // A non-finite report is no report. + expect( + resolveBilledCost({ + harness: "pi", + model: "openai/gpt-5.4-mini", + usage, + reportedCostUsd: Number.NaN, + priceMap, + }).cost_source, + ).toBe("computed"); + }); + + it("retains the dated matched model/source for aliases and omits pricing on reported or unavailable cost", () => { + const computed = resolveBilledCost({ + harness: "mastra", + model: "xai/grok-4.5", + usage, + priceMap, + }); + expect(computed.cost_pricing).toEqual({ + as_of: "2026-08-31", + model: "spacexai/grok-4.5", + source: "test", + }); + expect( + resolveBilledCost({ + harness: "pi", + model: "openai/gpt-5.4-mini", + usage, + reportedCostUsd: 0, + priceMap, + }), + ).toEqual({ cost_usd: 0, cost_source: "reported", billing_channel: "pi_catalog" }); + expect( + resolveBilledCost({ harness: "cursor", model: "openai/gpt-5.4-mini", usage, priceMap }), + ).not.toHaveProperty("cost_pricing"); + expect( + resolveBilledCost({ + harness: "pi", + model: "openai/gpt-5.4-mini", + usage, + reportedCostUsd: -1, + priceMap, + }).cost_source, + ).toBe("computed"); + }); + + it.each(["eve", "mastra", "pi"])( + "keeps missing %s usage unavailable and explicit zero usage priceable", + (harness) => { + const empty = { inputTokens: 0, outputTokens: 0, totalTokens: 0 }; + const unknown = normalizeUsage({ harness, raw: empty }); + expect( + resolveBilledCost({ harness, model: "openai/gpt-5.4-mini", usage: unknown, priceMap }), + ).toEqual({ cost_source: "unavailable", billing_channel: "openai_api" }); + const reportedZero = normalizeUsage({ harness, raw: { ...empty, reported: true } }); + expect( + resolveBilledCost({ harness, model: "openai/gpt-5.4-mini", usage: reportedZero, priceMap }), + ).toMatchObject({ cost_source: "computed", cost_usd: 0 }); + }, + ); + + it("derives the provider from the configured id", () => { + expect(providerOf("gateway/openai/gpt-5.4-mini")).toBe("openai"); + expect(providerOf("codex/default")).toBe("openai"); + expect(providerOf("xai/grok-4.5")).toBe("xai"); + expect(providerOf("gpt-5.4-mini")).toBeUndefined(); + expect(providerOf(undefined)).toBeUndefined(); + }); +}); + +describe("model alias resolution", () => { + it.each([ + ["gateway/openai/gpt-5.4-mini", "openai/gpt-5.4-mini"], + ["gateway/gpt-5.4-mini", "openai/gpt-5.4-mini"], + ["codex/default", "openai/gpt-5.4-mini"], + ["gpt-5.4-mini", "openai/gpt-5.4-mini"], + ["anthropic/claude-sonnet-4-6", "anthropic/claude-sonnet-4.6"], + ["anthropic/claude-sonnet-4.6-20260101", "anthropic/claude-sonnet-4.6"], + ["xai/grok-4.5", "spacexai/grok-4.5"], + ["grok-4.5", "spacexai/grok-4.5"], + ["google/gemini-3-flash-preview", "google/gemini-3-flash"], + ])("resolves %s to %s", (model, expected) => { + expect(resolveModelPrice(model, priceMap)?.key).toBe(expected); + }); + + it("never borrows a sibling model's price", () => { + expect(resolveModelPrice("openai/gpt-5.4", priceMap)).toBeUndefined(); + expect(resolveModelPrice("anthropic/claude-sonnet-4", priceMap)).toBeUndefined(); + expect(resolveModelPrice("fx/default", priceMap)).toBeUndefined(); + }); + + it("does not match a bare name carried by several providers", () => { + const ambiguous: PriceMap = { + as_of: "x", + models: { + "a/model-1": { input_per_m: 1, cached_input_per_m: 1, output_per_m: 1, source: "t" }, + "b/model-1": { input_per_m: 2, cached_input_per_m: 2, output_per_m: 2, source: "t" }, + }, + }; + expect(resolveModelPrice("model-1", ambiguous)).toBeUndefined(); + expect(resolveModelPrice("b/model-1", ambiguous)?.key).toBe("b/model-1"); + }); + + it("orders candidates from the exact id to the bare name", () => { + expect(modelPriceCandidates("gateway/xai/grok-4-5")).toEqual([ + "xai/grok-4-5", + "spacexai/grok-4-5", + "x-ai/grok-4-5", + "xai/grok-4.5", + "spacexai/grok-4.5", + "x-ai/grok-4.5", + "grok-4-5", + "grok-4.5", + ]); + }); +}); + +describe("shipped price map", () => { + it("prices the curated set", () => { + const shipped = loadPriceMap(); + expect(shipped.as_of).toMatch(/^\d{4}-\d{2}-\d{2}$/u); + expect(resolveModelPrice("openai/gpt-5.4-mini", shipped)).toBeDefined(); + expect(resolveModelPrice("anthropic/claude-sonnet-4-6", shipped)).toBeDefined(); + expect(resolveModelPrice("xai/grok-4.5", shipped)).toBeDefined(); + // gpt-5.6 luna/terra/sol are on the public OpenAI price list (2026-09-02). + for (const model of ["openai/gpt-5.6-luna", "openai/gpt-5.6-terra", "openai/gpt-5.6-sol"]) { + expect(shipped.models[model]?.source, model).toMatch(/developers\.openai\.com/u); + expect(resolveModelPrice(model, shipped), model).toBeDefined(); + } + // Fable 5.1 cache hits are 0.025x base input (other Anthropic models 0.1x). + expect(shipped.models["anthropic/claude-fable-5-1"]).toMatchObject({ + input_per_m: 10, + cached_input_per_m: 0.25, + output_per_m: 50, + cache_write_input_per_m: 12.5, + }); + }); +}); diff --git a/packages/evals/tests/framework/evalSystemPrompt.test.ts b/packages/evals/tests/framework/evalSystemPrompt.test.ts new file mode 100644 index 000000000..b60567b07 --- /dev/null +++ b/packages/evals/tests/framework/evalSystemPrompt.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, it } from "vitest"; +import { EvalLogger } from "../../logger.js"; +import { EVAL_SYSTEM_PROMPT } from "../../framework/evalSystemPrompt.js"; +import { runExternalHarnessTask } from "../../framework/harnesses/externalRunner.js"; + +describe("shared evaluation policy", () => { + it.each(["native", "task_prefix"] as const)("dispatches policy once via %s", async (mode) => { + let request = ""; + const result = await runExternalHarnessTask({ + harness: "example", + implementation: { name: "sdk", version: 1, sdkVersion: "1.0.0" }, + plan: { + dataset: "webvoyager", + taskId: "fixture", + instruction: "Return the exact page title.", + startUrl: "https://example.test", + }, + logger: new EvalLogger(false), + resultContract: "marker", + fallbackErrorMessage: "missing result", + systemPromptMode: mode, + runSession: async (prompt, systemPrompt) => { + request = `${systemPrompt}\n${prompt}`; + expect(systemPrompt).toBe(mode === "native" ? EVAL_SYSTEM_PROMPT : ""); + return { + raw: {}, + resultText: '{"success":true,"finalAnswer":"Example"}', + transcriptText: "", + status: "completed", + usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 }, + metrics: {}, + }; + }, + toTrajectory: () => { + throw new Error("verifier not configured"); + }, + }); + expect(request.split(EVAL_SYSTEM_PROMPT)).toHaveLength(2); + expect(request).toContain("Return the exact page title."); + expect(result.harnessImplementation).toEqual({ name: "sdk", version: 1, sdkVersion: "1.0.0" }); + }); +}); diff --git a/packages/evals/tests/framework/experimentMetadata.test.ts b/packages/evals/tests/framework/experimentMetadata.test.ts new file mode 100644 index 000000000..fa4ed6ebe --- /dev/null +++ b/packages/evals/tests/framework/experimentMetadata.test.ts @@ -0,0 +1,61 @@ +import { describe, expect, it } from "vitest"; +import { buildExperimentMetadata } from "../../framework/runner.js"; +import type { Testcase } from "../../types/evals.js"; + +function row(meta: Record): Testcase { + return { + input: { name: "agent/hardbenchmark", modelName: "openai/gpt-5.4-mini" as never }, + name: "agent/hardbenchmark", + tags: [], + metadata: { model: "openai/gpt-5.4-mini", test: "t", ...meta } as never, + expected: true, + }; +} + +describe("buildExperimentMetadata", () => { + it("always carries tool surface and model for bench runs, derived from rows", () => { + const meta = buildExperimentMetadata({ + environment: "BROWSERBASE", + tier: "bench", + harness: "mastra", + testcases: [ + row({ toolSurface: "stagehand_facade", provider: "openai", dataset: "hardbenchmark" }), + row({ toolSurface: "stagehand_facade", provider: "openai", dataset: "hardbenchmark" }), + ], + }); + expect(meta).toMatchObject({ + environment: "BROWSERBASE", + tier: "bench", + harness: "mastra", + tool_surface: "stagehand_facade", + model: "openai/gpt-5.4-mini", + provider: "openai", + dataset: "hardbenchmark", + task_count: 2, + }); + }); + + it("lists several distinct surfaces or models instead of dropping them", () => { + const meta = buildExperimentMetadata({ + environment: "LOCAL", + tier: "bench", + testcases: [ + row({ toolSurface: "stagehand_facade", model: "a/x" }), + row({ toolSurface: "stagehand_facade_legacy", model: "b/y" }), + ], + }); + expect(meta.tool_surface).toEqual(["stagehand_facade", "stagehand_facade_legacy"]); + expect(meta.model).toEqual(["a/x", "b/y"]); + }); + + it("prefers explicit core surface / model override and omits core placeholder models", () => { + const meta = buildExperimentMetadata({ + environment: "LOCAL", + tier: "core", + coreToolSurface: "understudy_code", + testcases: [row({ model: "none" })], + }); + expect(meta.tool_surface).toBe("understudy_code"); + expect(meta).not.toHaveProperty("model"); + }); +}); diff --git a/packages/evals/tests/framework/externalRunner.test.ts b/packages/evals/tests/framework/externalRunner.test.ts index 6fbfbda99..a85b0cad5 100644 --- a/packages/evals/tests/framework/externalRunner.test.ts +++ b/packages/evals/tests/framework/externalRunner.test.ts @@ -3,16 +3,29 @@ import { EvalLogger } from "../../logger.js"; import type { ExternalHarnessTaskPlan } from "../../framework/externalHarnessPlan.js"; import { buildExternalHarnessPrompt, + buildFacadeToolCallMetrics, buildNormalizedHarnessMetrics, + buildTimingMetrics, + buildUsageCostMetrics, + deriveTerminationReason, legacyHarnessFieldPrefix, parseEvalResult, + resolveFinalAnswer, runExternalHarnessTask, + stripEmbeddedEvalReports, } from "../../framework/harnesses/externalRunner.js"; +import { buildTrajectory } from "../../framework/harnesses/trajectoryAdapter.js"; +import { EVAL_SYSTEM_PROMPT } from "../../framework/evalSystemPrompt.js"; + +const verifierState = vi.hoisted(() => ({ + result: undefined as Record | undefined, +})); vi.mock("stagehand-v3", async (importOriginal) => { const mod = await importOriginal(); class FakeV3Evaluator { async verify() { + if (verifierState.result) return verifierState.result; throw new Error("fake verifier unavailable"); } } @@ -42,6 +55,119 @@ const plan: ExternalHarnessTaskPlan = { }; describe("external harness runner", () => { + it.each([true, false])( + "keeps execution loss separate from verified completion=%s", + async (outcomeSuccess) => { + verifierState.result = { + outcomeSuccess, + processScore: outcomeSuccess ? 1 : 0, + perCriterion: [{ criterion: "title", maxPoints: 1, earnedPoints: outcomeSuccess ? 1 : 0 }], + }; + try { + const result = await runExternalHarnessTask({ + harness: "example", + plan, + logger: new EvalLogger(false), + resultContract: "marker", + fallbackErrorMessage: "incomplete", + implementation: { name: "sdk", version: 1 }, + toolAdapter: { + browserSessionLoss: () => ({ + cause: "CDP connection closed", + timestamp: new Date().toISOString(), + }), + observedToolMatcher: (name) => name === "stagehand.run", + }, + verifier: { + v3: {} as never, + dataset: "test", + taskSpec: { + id: "fixture", + instruction: "Read title", + precomputedRubric: { + items: [ + { criterion: "title", description: "Read the correct title", maxPoints: 1 }, + ], + }, + }, + }, + runSession: async () => ({ + raw: {}, + resultText: '{"success":true,"finalAnswer":"Example"}', + transcriptText: "", + status: "completed", + usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 }, + metrics: {}, + }), + toTrajectory: ({ status }, taskSpec) => + buildTrajectory({ + taskSpec, + status, + finalAnswer: "Example", + toolCalls: [ + { + name: "stagehand.run", + args: { code: "return page.title()" }, + result: "Example", + ok: true, + }, + ], + }), + }); + expect(result.verifierError).toBeUndefined(); + expect(result._success).toBe(outcomeSuccess); + expect(result.harnessStatus).toBe("sdk_error"); + expect(result.terminationReason).toBe("browser_session_lost"); + expect(result.harnessImplementation).toEqual({ name: "sdk", version: 1 }); + } finally { + verifierState.result = undefined; + } + }, + ); + + it.each([ + ["marker", "native"], + ["structured_output", "native"], + ["marker", "task_prefix"], + ["structured_output", "task_prefix"], + ] as const)("dispatches %s with eval policy via %s", async (resultContract, systemPromptMode) => { + const instruction = "Find the item and stop before the final purchase."; + let dispatchedPrompt = ""; + let dispatchedSystemPrompt = ""; + await runExternalHarnessTask({ + harness: "test", + plan: { ...plan, instruction }, + logger: new EvalLogger(false), + resultContract, + systemPromptMode, + toolAdapter: { promptInstructions: "Use the mounted browser." }, + fallbackErrorMessage: "missing result", + runSession: async (prompt, systemPrompt) => { + dispatchedPrompt = prompt; + dispatchedSystemPrompt = systemPrompt; + return { + raw: {}, + resultText: '{"success":true}', + transcriptText: "", + status: "completed", + usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 }, + metrics: {}, + }; + }, + toTrajectory: () => { + throw new Error("no verifier configured"); + }, + }); + expect(dispatchedSystemPrompt).toBe(systemPromptMode === "native" ? EVAL_SYSTEM_PROMPT : ""); + expect(dispatchedPrompt.split(EVAL_SYSTEM_PROMPT)).toHaveLength( + systemPromptMode === "native" ? 1 : 2, + ); + expect(dispatchedPrompt).toContain(`Instruction:\n${instruction}`); + expect(dispatchedPrompt).toContain(plan.startUrl); + expect(dispatchedPrompt).toContain("Use the mounted browser."); + expect(dispatchedPrompt.match(/At the end,/g)).toHaveLength(1); + }); + it("parses the last result marker", () => { expect( parseEvalResult( @@ -82,6 +208,39 @@ describe("external harness runner", () => { expect(parseEvalResult("not json")).toEqual({ success: false, raw: "not json" }); }); + it("parses the last report-shaped object trailing free-form narration", () => { + const raw = [ + "I’ll open Imgur, inspect the meme, and report back.", + 'The tool returned {"url":"https://imgur.com"} so I continued.', + '{"success":true,"summary":"Inspected the meme.","finalAnswer":"A cat on a keyboard."}', + ].join("\n\n"); + expect(parseEvalResult(raw)).toMatchObject({ + success: true, + summary: "Inspected the meme.", + finalAnswer: "A cat on a keyboard.", + }); + // A report quoted mid-prose, with more prose after it, is not the conclusion. + expect(parseEvalResult(`${raw}\nthen I kept going`).success).toBe(false); + }); + + it("resolves the final answer from the report, else from the last message minus eval report envelopes", () => { + expect(resolveFinalAnswer({ finalAnswer: "42" }, "ignored")).toBe("42"); + expect( + resolveFinalAnswer({}, 'The answer is on the page.\n\n{"success":true,"summary":"reported"}'), + ).toBe("The answer is on the page."); + expect(resolveFinalAnswer({}, '{"only":"json"}')).toBe('{"only":"json"}'); + for (const deliverable of [ + '{"success":true,"order_id":"123"}', + '{"success":true,"summary":"Order shipped","order_id":"123"}', + '{"success":true}', + ]) + expect(resolveFinalAnswer({}, deliverable)).toBe(deliverable); + expect(parseEvalResult('{"success":true}').success).toBe(true); + + expect(resolveFinalAnswer({}, undefined)).toBeUndefined(); + expect(stripEmbeddedEvalReports("keep {not json} too")).toBe("keep {not json} too"); + }); + it("builds each result-contract tail", () => { const marker = buildExternalHarnessPrompt({ plan, resultContract: "marker" }); const structured = buildExternalHarnessPrompt({ @@ -134,6 +293,125 @@ describe("external harness runner", () => { expect(complete.harness_cost_usd.value).toBe(0.25); }); + it("does not let tool-output text relabel failed calls as trusted session loss", () => { + const lost = + "Browser session lost (CDP connection closed). The task cannot continue; report your final result now."; + const steps = [ + { actionName: "stagehand.run", actionArgs: {}, toolOutput: { ok: true, result: "x" } }, + { + actionName: "stagehand.run", + actionArgs: {}, + toolOutput: { ok: false, error: "bad xpath" }, + }, + { + actionName: "stagehand.run", + actionArgs: {}, + toolOutput: { ok: false, result: lost, error: lost }, + }, + { actionName: "stagehand.snapshot", actionArgs: {}, toolOutput: { ok: false, error: lost } }, + ]; + expect( + buildFacadeToolCallMetrics({ steps } as never, (name) => name.startsWith("stagehand.")), + ).toEqual({ + facade_tool_calls: { count: 1, value: 4 }, + facade_tool_call_failures: { count: 1, value: 3 }, + }); + }); + + it("records browser_session_lost as an SDK error even when the agent self-reports success", async () => { + const logger = new EvalLogger(false); + const result = await runExternalHarnessTask({ + harness: "deepagents", + plan, + logger, + resultContract: "structured_output", + fallbackErrorMessage: "missing result", + toolAdapter: { + browserSessionLoss: () => ({ cause: "CDP connection closed", tool: "run" }), + }, + runSession: async () => ({ + raw: {}, + resultText: '{"success":true,"summary":"done","finalAnswer":"326 E 110th St"}', + transcriptText: "", + status: "completed", + usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 }, + metrics: {}, + }), + toTrajectory: () => { + throw new Error("not called without a verifier"); + }, + }); + + expect(result).toMatchObject({ + _success: false, + error: "Browser session lost (CDP connection closed)", + finalAnswer: "326 E 110th St", + harnessStatus: "sdk_error", + harnessStopReason: "browser_session_lost", + deepagentsStatus: "sdk_error", + deepagentsStopReason: "browser_session_lost", + }); + expect(logger.getLogs().some((line) => line.message.includes("browser session lost"))).toBe( + true, + ); + }); + + it("counts facade tool calls and their failures from the trajectory", () => { + const steps = [ + { actionName: "stagehand.run", actionArgs: {}, toolOutput: { ok: true, result: "x" } }, + { + actionName: "stagehand.run", + actionArgs: {}, + toolOutput: { ok: false, result: "cancelled" }, + }, + { actionName: "node_repl.js", actionArgs: {}, toolOutput: { ok: true, result: "y" } }, + { actionName: "stagehand.snapshot", actionArgs: {} }, + ]; + expect( + buildFacadeToolCallMetrics({ steps } as never, (name) => name.startsWith("stagehand.")), + ).toEqual({ + facade_tool_calls: { count: 1, value: 3 }, + facade_tool_call_failures: { count: 1, value: 1 }, + }); + }); + + it("records effective policy channel, budget units, and requested effort", async () => { + const result = await runExternalHarnessTask({ + harness: "example", + plan, + logger: new EvalLogger(false), + resultContract: "marker", + fallbackErrorMessage: "missing result", + systemPromptMode: "native", + stepBudget: 100, + stepBudgetUnit: "tool_calls", + configuration: { requestedReasoningEffort: "high" }, + runSession: async (_prompt, policy) => { + expect(policy).toBe(EVAL_SYSTEM_PROMPT); + return { + raw: {}, + resultText: 'EVAL_RESULT: {"success":true,"finalAnswer":"done"}', + transcriptText: "", + status: "completed", + usage: { inputTokens: 0, outputTokens: 0, totalTokens: 0, reported: false }, + metrics: {}, + }; + }, + toTrajectory: () => { + throw new Error("no verifier"); + }, + }); + expect(result.harnessConfiguration).toEqual({ + evalPolicyVersion: 1, + systemPromptMode: "native", + stepBudget: 100, + stepBudgetUnit: "tool_calls", + requestedReasoningEffort: "high", + }); + expect(result.usageConvention).toBe("unreported"); + expect(result.cost_usd).toBeUndefined(); + }); + it("assembles normalized and deprecated task-result fields", async () => { const result = await runExternalHarnessTask({ harness: "claude_code", @@ -307,6 +585,34 @@ describe("external harness runner", () => { expect(result._success).toBe(true); expect(result.harnessStopReason).toBe("maximum turn budget reached"); + expect(result.terminationReason).toBe("step_budget"); + }); + + it("derives why a run ended from its status and stop reason", () => { + expect(deriveTerminationReason({ status: "completed" })).toBe("completed"); + expect( + deriveTerminationReason({ + status: "max_turns", + stopReason: "tool step budget exhausted (75 steps)", + }), + ).toBe("step_budget"); + expect( + deriveTerminationReason({ status: "sdk_error", stopReason: "browser_session_lost" }), + ).toBe("browser_session_lost"); + expect(deriveTerminationReason({ status: "sdk_error", stopReason: "aborted" })).toBe("aborted"); + expect( + deriveTerminationReason({ + status: "sdk_error", + stopReason: "This operation was aborted", + }), + ).toBe("aborted"); + expect(deriveTerminationReason({ status: "sdk_error", stopReason: "interrupted" })).toBe( + "aborted", + ); + expect(deriveTerminationReason({ status: "sdk_error", stopReason: "ECONNRESET" })).toBe( + "sdk_error", + ); + expect(deriveTerminationReason({ status: "sdk_error" })).toBe("sdk_error"); }); it("bounds hanging evidence capture before verifier fallback", async () => { @@ -351,7 +657,8 @@ describe("external harness runner", () => { toTrajectory: () => ({}) as never, }); - expect(result._success).toBe(true); + expect(result._success).toBe(false); + expect(result.agentReportedSuccess).toBe(true); expect(result.verifierError).toBeDefined(); expect(captureInvocations).toBe(1); expect(drainInvocations).toBe(1); @@ -361,6 +668,64 @@ describe("external harness runner", () => { } }); + it("ships the normalized step trace in the row logs once the trajectory exists", async () => { + const logger = new EvalLogger(false); + logger.log({ category: "codex", message: "tool-call-delta event", level: 2 }); + const result = await runExternalHarnessTask({ + harness: "codex", + plan, + logger, + toolAdapter: { observedToolMatcher: (name) => name.startsWith("stagehand.") }, + verifier: { + v3: {} as never, + taskSpec: { id: "wv-1", instruction: plan.instruction, precomputedRubric: {} as never }, + dataset: "webvoyager", + }, + resultContract: "structured_output", + fallbackErrorMessage: "missing result", + runSession: async () => ({ + raw: {}, + resultText: '{"success":true,"summary":"done","finalAnswer":"ok"}', + transcriptText: "", + status: "completed", + usage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 }, + metrics: {}, + }), + toTrajectory: (_input, taskSpec) => + buildTrajectory({ + taskSpec, + toolCalls: [ + { + name: "stagehand.run", + args: { code: "return page.title()" }, + result: "Example", + ok: true, + reasoning: "read the title", + }, + { name: "bash", args: { command: "echo hi" }, result: "hi", ok: true }, + ], + }), + }); + + const messages = (result.logs ?? []).map((line) => line.message); + expect(messages).toEqual([ + expect.stringContaining("cost unavailable for codex on (unknown model)"), + "step 1 · think · read the title", + "step 1 · run · ok · return page.title() → Example", + "step 2 · bash · ok · echo hi → hi", + "summary · done", + "answer · ok", + expect.stringMatching( + /^result · completed · steps=2 · facade_calls=1 · in=10 \(cached 0\) out=5 · agent=\d+\.\ds$/u, + ), + expect.stringContaining("verifier integration failed"), + expect.stringMatching( + /^timing · agent=\d+\.\ds · evidence=\d+\.\ds · verifier=\d+\.\ds · total=\d+\.\ds$/u, + ), + ]); + expect(result.metrics).toMatchObject({ facade_tool_calls: { count: 1, value: 1 } }); + }); + it.each([ ["never resolves", () => new Promise(() => {})], ["rejects", () => Promise.reject(new Error("drain failed"))], @@ -411,7 +776,8 @@ describe("external harness runner", () => { }, }); - expect(result._success).toBe(true); + expect(result._success).toBe(false); + expect(result.verifierError).toBeDefined(); expect(trajectoryInput?.finalObservation).toBe(finalObservation); expect(trajectoryInput?.stepObservations).toBeUndefined(); expect(captureInvocations).toBe(1); @@ -421,4 +787,339 @@ describe("external harness runner", () => { else process.env.EVAL_CAPTURE_EVIDENCE_TIMEOUT_MS = previous; } }); + + it("splits agent, evidence and verifier wall-clock into separate metrics", async () => { + const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + const result = await runExternalHarnessTask({ + harness: "codex", + plan, + logger: new EvalLogger(false), + toolAdapter: { + captureEvidence: async () => { + await delay(30); + return { url: "https://example.com" } as never; + }, + }, + verifier: { + v3: {} as never, + taskSpec: { id: "wv-1", instruction: plan.instruction, precomputedRubric: {} as never }, + dataset: "webvoyager", + }, + resultContract: "structured_output", + fallbackErrorMessage: "missing result", + runSession: async () => { + await delay(50); + return { + raw: {}, + resultText: '{"success":true,"summary":"done","finalAnswer":"ok"}', + transcriptText: "", + status: "completed", + usage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 }, + metrics: {}, + }; + }, + toTrajectory: (_input, taskSpec) => buildTrajectory({ taskSpec, toolCalls: [] }), + }); + const metrics = result.metrics as Record; + + expect(metrics.agent_wall_ms.value).toBeGreaterThanOrEqual(45); + expect(metrics.evidence_ms.value).toBeGreaterThanOrEqual(25); + expect(metrics.verifier_wall_ms.value).toBeGreaterThanOrEqual(0); + expect(metrics.total_wall_ms.value).toBeCloseTo( + metrics.agent_wall_ms.value + metrics.evidence_ms.value + metrics.verifier_wall_ms.value, + 3, + ); + expect(result.agent_wall_ms).toBe(Math.round(metrics.agent_wall_ms.value)); + }); + + it("reports total_wall_ms as the agent time alone when no verifier runs", async () => { + const result = await runExternalHarnessTask({ + harness: "codex", + plan, + logger: new EvalLogger(false), + resultContract: "marker", + fallbackErrorMessage: "missing result", + runSession: async () => ({ + raw: {}, + resultText: 'EVAL_RESULT: {"success":true}', + transcriptText: "", + status: "completed", + usage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 }, + metrics: {}, + }), + toTrajectory: () => { + throw new Error("not called without a verifier"); + }, + }); + const metrics = result.metrics as Record; + expect(metrics.total_wall_ms.value).toBe(metrics.agent_wall_ms.value); + expect(metrics.verifier_wall_ms).toBeUndefined(); + }); + + it("sums the timing split into total_wall_ms", () => { + expect(buildTimingMetrics({ agentWallMs: 1000, evidenceMs: 200, verifierWallMs: 300 })).toEqual( + { + agent_wall_ms: { count: 1, value: 1000 }, + evidence_ms: { count: 1, value: 200 }, + verifier_wall_ms: { count: 1, value: 300 }, + total_wall_ms: { count: 1, value: 1500 }, + }, + ); + }); + + it("emits normalized usage and the harness-reported bill next to the harness-native metrics", async () => { + const logger = new EvalLogger(false); + const result = await runExternalHarnessTask({ + harness: "claude_code", + plan, + model: "anthropic/claude-sonnet-4-6", + logger, + resultContract: "marker", + fallbackErrorMessage: "missing result", + runSession: async () => ({ + raw: {}, + resultText: 'EVAL_RESULT: {"success":true}', + transcriptText: "", + status: "completed", + usage: { + inputTokens: 1_000_000, + cachedInputTokens: 2_000_000, + cacheCreationInputTokens: 0, + outputTokens: 100_000, + totalTokens: 3_100_000, + }, + costUsd: 4.2, + metrics: {}, + }), + toTrajectory: () => { + throw new Error("not called without a verifier"); + }, + }); + const metrics = result.metrics as Record; + + // Anthropic convention: cache reads sit outside input_tokens. + expect(metrics.usage_input_total.value).toBe(3_000_000); + expect(metrics.usage_input_cached.value).toBe(2_000_000); + expect(metrics.usage_output.value).toBe(100_000); + expect(metrics.usage_reasoning.value).toBe(0); + // Legacy metrics stay untouched for existing dashboards. + expect(metrics.harness_input_tokens.value).toBe(1_000_000); + expect(metrics.harness_cost_usd.value).toBe(4.2); + // The reported bill is the cost column. + expect(metrics.cost_usd.value).toBe(4.2); + expect(result).toMatchObject({ + cost_source: "reported", + billing_channel: "anthropic_api", + cost_usd: 4.2, + }); + expect(logger.getLogs().some((line) => line.category === "cost")).toBe(false); + }); + + it("computes a direct-API harness's bill at provider list price when nothing was reported", async () => { + const logger = new EvalLogger(false); + const result = await runExternalHarnessTask({ + harness: "codex", + plan, + model: "openai/gpt-5.4-mini", + logger, + resultContract: "marker", + fallbackErrorMessage: "missing result", + runSession: async () => ({ + raw: {}, + resultText: 'EVAL_RESULT: {"success":true}', + transcriptText: "", + status: "completed", + usage: { inputTokens: 1_000_000, outputTokens: 0, totalTokens: 1_000_000 }, + metrics: {}, + }), + toTrajectory: () => { + throw new Error("not called without a verifier"); + }, + }); + const metrics = result.metrics as Record; + + // gpt-5.4-mini list price $0.75/M uncached input. + expect(metrics.cost_usd.value).toBeCloseTo(0.75, 6); + expect(result).toMatchObject({ + cost_source: "computed", + billing_channel: "openai_api", + cost_pricing: { + as_of: expect.any(String), + model: expect.any(String), + source: expect.any(String), + }, + }); + expect(logger.getLogs().some((line) => line.category === "cost")).toBe(false); + }); + + it("leaves cost absent and names the model and channel when the bill is unavailable", async () => { + const logger = new EvalLogger(false); + const result = await runExternalHarnessTask({ + harness: "codex", + plan, + model: "openai/gpt-unpriced-fixture", + logger, + resultContract: "marker", + fallbackErrorMessage: "missing result", + runSession: async () => ({ + raw: {}, + resultText: 'EVAL_RESULT: {"success":true}', + transcriptText: "", + status: "completed", + usage: { inputTokens: 100, cachedInputTokens: 40, outputTokens: 5, totalTokens: 105 }, + metrics: {}, + }), + toTrajectory: () => { + throw new Error("not called without a verifier"); + }, + }); + const metrics = result.metrics as Record; + + expect(metrics.usage_input_total.value).toBe(100); + expect(metrics.usage_input_cached.value).toBe(40); + expect(metrics.cost_usd).toBeUndefined(); + expect(result.cost_source).toBe("unavailable"); + expect(result.billing_channel).toBe("openai_api"); + expect(result.cost_usd).toBeUndefined(); + const costLine = logger.getLogs().find((line) => line.category === "cost"); + expect(costLine?.level).toBe(1); + expect(costLine?.message).toContain("openai/gpt-unpriced-fixture"); + expect(costLine?.message).toContain("openai_api"); + }); + + it("shows the billed cost and its source on the timing line", async () => { + const logger = new EvalLogger(false); + const result = await runExternalHarnessTask({ + harness: "codex", + plan, + model: "openai/gpt-5.4-mini", + logger, + verifier: { + v3: {} as never, + taskSpec: { id: "wv-1", instruction: plan.instruction, precomputedRubric: {} as never }, + dataset: "webvoyager", + }, + resultContract: "structured_output", + fallbackErrorMessage: "missing result", + runSession: async () => ({ + raw: {}, + resultText: '{"success":true,"summary":"done","finalAnswer":"ok"}', + transcriptText: "", + status: "completed", + usage: { inputTokens: 1_000_000, outputTokens: 0, totalTokens: 1_000_000 }, + metrics: {}, + }), + toTrajectory: (_input, taskSpec) => buildTrajectory({ taskSpec, toolCalls: [] }), + }); + const timing = result.logs?.find((line) => line.message.startsWith("timing")); + expect(timing?.message).toContain("cost=$0.75 (computed)"); + }); + + it("builds usage/cost metrics without inventing a zero-dollar estimate", () => { + const usage = { + input_total: 10, + input_cached: 4, + input_cache_write: 0, + input_uncached: 6, + output: 2, + reasoning: 1, + reasoning_in_output: true, + convention: "openai_cached_subset" as const, + }; + expect( + buildUsageCostMetrics(usage, { cost_source: "unavailable", billing_channel: "openai_api" }), + ).toEqual({ + usage_input_total: { count: 1, value: 10 }, + usage_input_cached: { count: 1, value: 4 }, + usage_output: { count: 1, value: 2 }, + usage_reasoning: { count: 1, value: 1 }, + }); + expect( + buildUsageCostMetrics(usage, { + cost_source: "computed", + cost_usd: 0.5, + billing_channel: "openai_api", + }), + ).toEqual({ + usage_input_total: { count: 1, value: 10 }, + usage_input_cached: { count: 1, value: 4 }, + usage_output: { count: 1, value: 2 }, + usage_reasoning: { count: 1, value: 1 }, + cost_usd: { count: 1, value: 0.5 }, + }); + // Unreported usage carries no usage_* metrics: zeros would read as a free run. + expect( + buildUsageCostMetrics( + { + ...usage, + input_total: 0, + input_cached: 0, + input_uncached: 0, + output: 0, + reasoning: 0, + convention: "unreported", + }, + { cost_source: "unavailable", billing_channel: "subscription" }, + ), + ).toEqual({}); + }); +}); + +it("keeps task JSON as the final answer and passes it to trajectory conversion", async () => { + const answer = JSON.stringify({ products: [{ name: "example", price: 19 }] }); + let trajectoryAnswer: string | undefined; + const result = await runExternalHarnessTask({ + harness: "codex", + plan, + logger: new EvalLogger(false), + verifier: { + v3: {} as never, + taskSpec: { id: "json-task", instruction: plan.instruction, precomputedRubric: {} as never }, + dataset: "webvoyager", + }, + resultContract: "structured_output", + fallbackErrorMessage: "missing result", + runSession: async () => ({ + raw: {}, + resultText: answer, + transcriptText: "", + status: "completed", + usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 }, + metrics: {}, + }), + toTrajectory: ({ parsed }, taskSpec) => { + trajectoryAnswer = parsed.finalAnswer; + return buildTrajectory({ taskSpec, toolCalls: [], finalAnswer: parsed.finalAnswer }); + }, + }); + expect(result.finalAnswer).toBe(answer); + expect(trajectoryAnswer).toBe(answer); +}); + +it("sanitizes SDK stop reasons before emitting verifier trajectory traces", async () => { + const logger = new EvalLogger(false); + await runExternalHarnessTask({ + harness: "codex", + plan, + logger, + verifier: { + v3: {} as never, + taskSpec: { id: "trace-task", instruction: plan.instruction, precomputedRubric: {} as never }, + dataset: "webvoyager", + }, + resultContract: "structured_output", + fallbackErrorMessage: "missing result", + runSession: async () => ({ + raw: {}, + resultText: "", + transcriptText: "", + status: "sdk_error", + stopReason: "request failed https://provider.example?apiKey=secret-query-value", + usage: { inputTokens: 0, outputTokens: 0, totalTokens: 0 }, + metrics: {}, + }), + toTrajectory: (_input, taskSpec) => buildTrajectory({ taskSpec, toolCalls: [] }), + }); + expect(logger.getLogs().some((line) => line.category === "trace")).toBe(true); + expect(JSON.stringify(logger.getLogs())).not.toContain("secret-query-value"); }); diff --git a/packages/evals/tests/framework/reasoningSummary.test.ts b/packages/evals/tests/framework/reasoningSummary.test.ts new file mode 100644 index 000000000..4cf9e2342 --- /dev/null +++ b/packages/evals/tests/framework/reasoningSummary.test.ts @@ -0,0 +1,39 @@ +import { describe, expect, it } from "vitest"; +import { + DEFAULT_REASONING_SUMMARY, + isOpenAiModel, + openAiReasoningProviderOptions, + readReasoningSummary, +} from "../../framework/reasoningSummary.js"; + +describe("reasoning summary switch", () => { + it("is on by default and honours every documented spelling of off", () => { + expect(readReasoningSummary({})).toBe(DEFAULT_REASONING_SUMMARY); + expect(readReasoningSummary({ EVAL_REASONING_SUMMARY: "auto" })).toBe("auto"); + expect(readReasoningSummary({ EVAL_REASONING_SUMMARY: "Concise" })).toBe("concise"); + for (const off of ["off", "none", "false", "0"]) { + expect(readReasoningSummary({ EVAL_REASONING_SUMMARY: off })).toBeUndefined(); + } + expect(readReasoningSummary({ EVAL_REASONING_SUMMARY: "verbose" })).toBe( + DEFAULT_REASONING_SUMMARY, + ); + }); + + it.each(["openai/gpt-4.1-mini", "gpt-4o", "text-embedding-3-small", "openai/gpt-5-chat-latest"])( + "does not request reasoning on %s", + (model) => expect(openAiReasoningProviderOptions(model, {})).toBeUndefined(), + ); + + it("only asks OpenAI models for summaries", () => { + expect(isOpenAiModel("openai/gpt-5.6-luna")).toBe(true); + expect(isOpenAiModel("gpt-5.4-mini")).toBe(true); + expect(isOpenAiModel("anthropic/claude-sonnet-4-6")).toBe(false); + expect(openAiReasoningProviderOptions("openai/gpt-5.6-luna", {})).toEqual({ + openai: { reasoningSummary: "detailed" }, + }); + expect(openAiReasoningProviderOptions("anthropic/claude-sonnet-4-6", {})).toBeUndefined(); + expect( + openAiReasoningProviderOptions("openai/gpt-5.6-luna", { EVAL_REASONING_SUMMARY: "off" }), + ).toBeUndefined(); + }); +}); diff --git a/packages/evals/tests/framework/stepBudget.test.ts b/packages/evals/tests/framework/stepBudget.test.ts new file mode 100644 index 000000000..6def51a7a --- /dev/null +++ b/packages/evals/tests/framework/stepBudget.test.ts @@ -0,0 +1,68 @@ +import { describe, expect, it } from "vitest"; +import { DATASET_STEP_BUDGETS, resolveStepBudget } from "../../framework/stepBudget.js"; + +describe("resolveStepBudget", () => { + it("prefers the harness-specific env key over everything else", () => { + expect( + resolveStepBudget({ + harnessEnvKey: "EVAL_CODEX_MAX_STEPS", + dataset: "hardbenchmark", + harnessDefault: 100, + env: { EVAL_CODEX_MAX_STEPS: "12", AGENT_EVAL_MAX_STEPS: "34" }, + }), + ).toBe(12); + }); + + it("falls back to AGENT_EVAL_MAX_STEPS before the dataset budget", () => { + expect( + resolveStepBudget({ + harnessEnvKey: "EVAL_CODEX_MAX_STEPS", + dataset: "hardbenchmark", + harnessDefault: 100, + env: { AGENT_EVAL_MAX_STEPS: "34" }, + }), + ).toBe(34); + }); + + it("applies the dataset budget when no env override is set", () => { + expect(DATASET_STEP_BUDGETS.hardbenchmark).toBe(100); + expect( + resolveStepBudget({ + harnessEnvKey: "EVAL_EVE_MAX_STEPS", + dataset: "hardbenchmark", + harnessDefault: 50, + env: {}, + }), + ).toBe(100); + }); + + it("keeps the harness default for datasets without a budget", () => { + expect( + resolveStepBudget({ + harnessEnvKey: "EVAL_EVE_MAX_STEPS", + dataset: "webvoyager", + harnessDefault: 50, + env: {}, + }), + ).toBe(50); + expect( + resolveStepBudget({ + harnessEnvKey: "EVAL_CLAUDE_CODE_MAX_TURNS", + dataset: undefined, + harnessDefault: 50, + env: {}, + }), + ).toBe(50); + }); + + it("ignores non-positive and non-numeric env values", () => { + expect( + resolveStepBudget({ + harnessEnvKey: "EVAL_FX_MAX_STEPS", + dataset: "hardbenchmark", + harnessDefault: 60, + env: { EVAL_FX_MAX_STEPS: "0", AGENT_EVAL_MAX_STEPS: "lots" }, + }), + ).toBe(100); + }); +}); diff --git a/packages/evals/tests/framework/toSummaryResult.test.ts b/packages/evals/tests/framework/toSummaryResult.test.ts new file mode 100644 index 000000000..44ce0eccf --- /dev/null +++ b/packages/evals/tests/framework/toSummaryResult.test.ts @@ -0,0 +1,44 @@ +import { describe, expect, it } from "vitest"; +import { toSummaryResult } from "../../framework/runner.js"; +import type { EvalInput } from "../../types/evals.js"; + +const input = { name: "agent/hardbenchmark", modelName: "google/gemini-3.5-flash" } as EvalInput; + +describe("toSummaryResult", () => { + it("keeps object outputs and derives the score", () => { + const row = toSummaryResult({ + input, + output: { _success: true, steps: 4 }, + metadata: { categories: ["navigation", 3] }, + }); + expect(row).toEqual({ + input, + output: { _success: true, steps: 4 }, + name: "agent/hardbenchmark", + score: 1, + categories: ["navigation"], + }); + }); + + it("wraps boolean outputs", () => { + expect(toSummaryResult({ input, output: false }).output).toEqual({ _success: false }); + }); + + it("treats a Braintrust row without output as a failed row carrying the error", () => { + const row = toSummaryResult({ input, output: undefined, error: new Error("span failed") }); + expect(row.score).toBe(0); + expect(row.output).toEqual({ _success: false, error: "span failed" }); + expect(row.name).toBe("agent/hardbenchmark"); + }); + + it("explains a missing output when Braintrust reports no error either", () => { + expect(toSummaryResult({ input }).output).toEqual({ + _success: false, + error: "Braintrust reported no output for this task", + }); + expect(toSummaryResult({ input, output: null, error: "boom" }).output).toEqual({ + _success: false, + error: "boom", + }); + }); +}); diff --git a/packages/evals/tests/framework/toolSurfaceResolution.test.ts b/packages/evals/tests/framework/toolSurfaceResolution.test.ts index 385ecd4b1..e47084213 100644 --- a/packages/evals/tests/framework/toolSurfaceResolution.test.ts +++ b/packages/evals/tests/framework/toolSurfaceResolution.test.ts @@ -12,8 +12,8 @@ describe("tool surface resolution", () => { expect(resolveToolSurface(stagehandHarness, "understudy_code")).toBe("understudy_code"); }); - it("defaults to the first supported surface and accepts supported requests", () => { - expect(resolveToolSurface(claudeCodeHarness)).toBe("browse_cli"); + it("defaults to the facade when supported and accepts supported requests", () => { + expect(resolveToolSurface(claudeCodeHarness)).toBe("stagehand_facade"); expect(resolveToolSurface(claudeCodeHarness, "cdp_code")).toBe("cdp_code"); }); diff --git a/packages/evals/tests/framework/traceLog.test.ts b/packages/evals/tests/framework/traceLog.test.ts new file mode 100644 index 000000000..9b57e75d4 --- /dev/null +++ b/packages/evals/tests/framework/traceLog.test.ts @@ -0,0 +1,223 @@ +import { describe, expect, it } from "vitest"; +import type { Trajectory } from "stagehand-v3"; +import { + buildTrajectoryTraceLines, + emitTrajectoryTrace, + shortToolName, + TRACE_AUXILIARY_MAX_CHARS, + type TrajectoryTraceInput, +} from "../../framework/harnesses/traceLog.js"; +import { buildTrajectory } from "../../framework/harnesses/trajectoryAdapter.js"; + +const taskSpec = { id: "wv-1", instruction: "Find the title", initUrl: "https://example.com" }; + +function threeStepTrajectory(): Trajectory { + return buildTrajectory({ + taskSpec, + toolCalls: [ + { + name: "mcp__stagehand__run", + args: { code: "await page.goto('https://example.com');\n return page.title();" }, + result: "Example Domain", + ok: true, + reasoning: "I should open the start URL first\nand read the title.", + }, + { + name: "stagehand.screenshot", + args: {}, + result: "Screenshot captured.", + ok: true, + images: [{ bytes: Buffer.alloc(42 * 1024), mediaType: "image/png" }], + }, + { + name: "stagehand_run", + args: { code: "await page.click('#nope')" }, + result: undefined, + ok: false, + error: "Timeout 30000ms exceeded waiting for #nope", + }, + ], + usage: { input_tokens: 100, output_tokens: 20 }, + }); +} + +const outcome: TrajectoryTraceInput["outcome"] = { + status: "completed", + stopReason: undefined, + usage: { inputTokens: 12345, outputTokens: 678, cachedInputTokens: 9000, totalTokens: 13023 }, +}; + +describe("trajectory trace log", () => { + it("normalizes FX MCP facade tool names", () => { + expect(shortToolName("mcp_stagehand_run")).toBe("run"); + expect(shortToolName("mcp_stagehand_browser_snapshot")).toBe("snapshot"); + }); + + it("emits one identical-shape line per step plus a result line", () => { + const lines = buildTrajectoryTraceLines({ + trajectory: threeStepTrajectory(), + outcome, + isFacadeTool: (name) => /run|screenshot/u.test(name), + }); + + expect(lines.map((line) => line.message)).toEqual([ + "step 1 · think · I should open the start URL first and read the title.", + "step 1 · run · ok · await page.goto('https://example.com'); return page.title(); → Example Domain", + "step 2 · screenshot · ok → [image 42 KB]", + "step 3 · run · ERR · await page.click('#nope') → Timeout 30000ms exceeded waiting for #nope", + "answer · (none — agent reported none)", + "result · completed · steps=3 · facade_calls=3 · in=12345 out=678 cached=9000", + ]); + expect(lines.every((line) => line.category === "trace")).toBe(true); + expect(lines.map((line) => line.level)).toEqual([1, 1, 1, 0, 1, 1]); + }); + + it("traces the agent's summary and final answer before the result line", () => { + const lines = buildTrajectoryTraceLines({ + trajectory: threeStepTrajectory(), + outcome, + report: { + success: true, + summary: "Opened the site and read the title.", + finalAnswer: "Example Domain\nsecond line " + "x".repeat(300), + }, + }); + const tail = lines.slice(-3).map((line) => line.message); + expect(tail[0]).toBe("summary · Opened the site and read the title."); + expect(tail[1].startsWith("answer · Example Domain second line xxx")).toBe(true); + expect(tail[1].length).toBeLessThanOrEqual("answer · ".length + 200 + 1); + expect(lines.at(-2)?.auxiliary?.answer?.value).toContain("x".repeat(300)); + expect(tail[2].startsWith("result · completed")).toBe(true); + }); + + it("states a missing answer with the stop status, as an error line", () => { + const lines = buildTrajectoryTraceLines({ + trajectory: threeStepTrajectory(), + outcome: { ...outcome, status: "max_turns", stopReason: "turn budget exhausted" }, + report: { success: false, summary: "", finalAnswer: "" }, + }); + const answer = lines.at(-2)!; + expect(answer.message).toBe("answer · (none — max_turns)"); + expect(answer.level).toBe(0); + }); + + it("keeps the full code and result in auxiliary", () => { + const [, run, , failed, , result] = buildTrajectoryTraceLines({ + trajectory: threeStepTrajectory(), + outcome, + }); + expect(run.auxiliary).toEqual({ + tool: { value: "mcp__stagehand__run", type: "string" }, + code: { + value: "await page.goto('https://example.com'); return page.title();", + type: "string", + }, + result: { value: "Example Domain", type: "string" }, + }); + expect(failed.auxiliary?.error).toEqual({ + value: "Timeout 30000ms exceeded waiting for #nope", + type: "string", + }); + expect(failed.auxiliary?.result).toBeUndefined(); + expect(result.auxiliary?.usage).toEqual({ + value: JSON.stringify(outcome.usage), + type: "object", + }); + expect(result.message).not.toContain("facade_calls"); + }); + + it("clips long code and results to a single line while capping auxiliary", () => { + const longCode = "await page.locator('x').click();\n".repeat(40); + const hugeResult = "y".repeat(TRACE_AUXILIARY_MAX_CHARS + 500); + const [line] = buildTrajectoryTraceLines({ + trajectory: buildTrajectory({ + taskSpec, + toolCalls: [{ name: "run", args: { code: longCode }, result: hugeResult, ok: true }], + }), + outcome, + }); + expect(line.message).not.toContain("\n"); + expect(line.message.length).toBeLessThan(450); + expect(line.message).toMatch(/… → y+…$/u); + expect(line.auxiliary?.result?.value).toHaveLength( + TRACE_AUXILIARY_MAX_CHARS + "…[truncated 500 chars]".length, + ); + }); + + it("summarizes snapshots by node count, object results as JSON, and stop reasons", () => { + const lines = buildTrajectoryTraceLines({ + trajectory: buildTrajectory({ + taskSpec, + toolCalls: [ + { + name: "stagehand.snapshot", + args: { includeIframes: true }, + result: "[1-1] RootWebArea: Example\n [1-2] link: More\n [1-3] button: Go", + ok: true, + }, + { + name: "mcp__stagehand__run", + args: { code: "return {a: 1}" }, + result: { a: 1 }, + ok: true, + }, + { name: "bash", args: { command: "ls -la" }, result: "total 0", ok: true }, + ], + }), + outcome: { ...outcome, status: "sdk_error", stopReason: "max turns\nreached" }, + }); + expect(lines.map((line) => line.message)).toEqual([ + 'step 1 · snapshot · ok · {"includeIframes":true} → [snapshot 3 nodes] [1-1] RootWebArea: Example', + 'step 2 · run · ok · return {a: 1} → {"a":1}', + "step 3 · bash · ok · ls -la → total 0", + "answer · (none — sdk_error)", + "result · sdk_error · max turns reached · steps=3 · in=12345 out=678 cached=9000", + ]); + }); + + it("prefers the normalized usage and agent wall-clock on the result line", () => { + const lines = buildTrajectoryTraceLines({ + trajectory: threeStepTrajectory(), + outcome, + usage: { + input_total: 21345, + input_cached: 9000, + input_cache_write: 0, + input_uncached: 12345, + output: 678, + reasoning: 0, + reasoning_in_output: true, + convention: "anthropic_cache_separate", + }, + agentWallMs: 42_049, + }); + const result = lines.at(-1)!; + expect(result.message).toBe( + "result · completed · steps=3 · in=21345 (cached 9000) out=678 · agent=42.0s", + ); + expect(JSON.parse(result.auxiliary!.normalized_usage!.value as string)).toMatchObject({ + convention: "anthropic_cache_separate", + }); + }); + + it("emits through an EvalLogger-compatible sink", () => { + const logged: string[] = []; + emitTrajectoryTrace( + { log: (line) => void logged.push(line.message) }, + { trajectory: threeStepTrajectory(), outcome }, + ); + expect(logged).toHaveLength(6); + expect(logged.at(-1)).toMatch(/^result · completed/u); + }); + + it("collapses harness-specific tool names onto the surface tool", () => { + expect(shortToolName("mcp__stagehand__run")).toBe("run"); + expect(shortToolName("mcp__stagehand_browser__run")).toBe("run"); + expect(shortToolName("stagehand.snapshot")).toBe("snapshot"); + expect(shortToolName("stagehand_screenshot")).toBe("screenshot"); + expect(shortToolName("stagehand_run")).toBe("run"); + expect(shortToolName("Bash")).toBe("Bash"); + expect(shortToolName("web_search")).toBe("web_search"); + expect(shortToolName("")).toBe("tool"); + }); +}); diff --git a/packages/evals/tests/framework/usageNormalization.test.ts b/packages/evals/tests/framework/usageNormalization.test.ts new file mode 100644 index 000000000..2b0b9092c --- /dev/null +++ b/packages/evals/tests/framework/usageNormalization.test.ts @@ -0,0 +1,279 @@ +import { describe, expect, it } from "vitest"; +import { + formatNormalizedUsage, + normalizeUsage, + usageConventionFor, +} from "../../framework/usageNormalization.js"; + +describe("normalizeUsage", () => { + it.each(["constructor", "__proto__", "toString"])( + "defaults safely for unknown harness %s", + (harness) => { + expect(usageConventionFor(harness)).toBe("openai_cached_subset"); + expect( + normalizeUsage({ harness, raw: { inputTokens: 10, outputTokens: 5, totalTokens: 15 } }), + ).toMatchObject({ input_total: 10, output: 5, convention: "openai_cached_subset" }); + }, + ); + + it("keeps FX reasoning outside output without duplicating cached input", () => { + const usage = normalizeUsage({ + harness: "fx", + raw: { + inputTokens: 1000, + cachedInputTokens: 600, + outputTokens: 200, + reasoningOutputTokens: 50, + totalTokens: 1250, + }, + }); + expect(usage).toMatchObject({ + input_total: 1000, + input_uncached: 400, + output: 200, + reasoning: 50, + reasoning_in_output: false, + }); + expect(usage.input_total + usage.output + usage.reasoning).toBe(1250); + }); + + it.each(["eve", "mastra", "pi", "codex", "cursor", "cursor_sdk"])( + "treats zero-filled %s telemetry with no presence flag as unknown", + (harness) => { + const raw = { inputTokens: 0, outputTokens: 0, totalTokens: 0 }; + expect(normalizeUsage({ harness, raw }).convention).toBe("unreported"); + expect(normalizeUsage({ harness, raw: { ...raw, reported: true } }).convention).toBe( + usageConventionFor(harness), + ); + }, + ); + + it("keeps explicit missing Cursor telemetry unknown even when stale counts are nonzero", () => { + const raw = { + inputTokens: 10, + cachedInputTokens: 100, + outputTokens: 20, + totalTokens: 130, + reported: false, + }; + expect(normalizeUsage({ harness: "cursor", raw }).convention).toBe("unreported"); + expect(normalizeUsage({ harness: "cursor", raw: { ...raw, reported: true } })).toMatchObject({ + input_total: 110, + convention: "uncached_only", + }); + }); + + it("uses cache-only counts only for conventions where cache is a separate billable bucket", () => { + const raw = { inputTokens: 0, outputTokens: 0, cachedInputTokens: 100, totalTokens: 100 }; + expect(normalizeUsage({ harness: "pi", raw })).toMatchObject({ + convention: "uncached_only", + input_total: 100, + }); + expect(normalizeUsage({ harness: "codex", raw }).convention).toBe("unreported"); + expect( + normalizeUsage({ + harness: "codex", + raw: { ...raw, cachedInputTokens: 0, reasoningOutputTokens: 100 }, + }).convention, + ).toBe("unreported"); + }); + + it("does not infer priced buckets from a total-only or invalid count", () => { + for (const inputTokens of [0, -1, NaN, Infinity]) { + expect( + normalizeUsage({ + harness: "mastra", + raw: { inputTokens, outputTokens: 0, totalTokens: 500 }, + }).convention, + ).toBe("unreported"); + } + }); + + it("treats cursor_sdk usage as unreported when no token buckets are exposed", () => { + expect(usageConventionFor("cursor_sdk")).toBe("uncached_only"); + expect( + normalizeUsage({ + harness: "cursor_sdk", + raw: { inputTokens: 0, outputTokens: 0, totalTokens: 0, reported: false }, + }).convention, + ).toBe("unreported"); + }); + + it("adds Cursor SDK cache reads and writes to its uncached input bucket", () => { + // @cursor/sdk 1.0.31 toTokenUsage computes total as the sum of these four + // buckets; reasoning is already part of output and must not be added again. + const raw = { + inputTokens: 40, + cachedInputTokens: 127_000, + cacheCreationInputTokens: 3_000, + outputTokens: 900, + reasoningOutputTokens: 120, + totalTokens: 130_940, + reported: true, + }; + const usage = normalizeUsage({ harness: "cursor_sdk", raw }); + expect(usage).toEqual({ + input_total: 130_040, + input_cached: 127_000, + input_cache_write: 3_000, + input_uncached: 40, + output: 900, + reasoning: 120, + reasoning_in_output: true, + convention: "uncached_only", + }); + expect(usage.input_total + usage.output).toBe(raw.totalTokens); + }); + + it("treats OpenAI-style cached tokens as a subset of input (codex, mastra, eve, deepagents, fx)", () => { + for (const harness of ["codex", "mastra", "eve", "deepagents"]) { + const usage = normalizeUsage({ + harness, + raw: { + inputTokens: 1000, + cachedInputTokens: 600, + outputTokens: 200, + reasoningOutputTokens: 50, + totalTokens: 1200, + }, + }); + expect(usage, harness).toEqual({ + input_total: 1000, + input_cached: 600, + input_cache_write: 0, + input_uncached: 400, + output: 200, + reasoning: 50, + reasoning_in_output: true, + convention: "openai_cached_subset", + }); + } + }); + + it("keeps cache writes inside the total for the subset convention (AI SDK anthropic via eve)", () => { + const usage = normalizeUsage({ + harness: "eve", + raw: { + inputTokens: 1000, + cachedInputTokens: 600, + cacheCreationInputTokens: 100, + outputTokens: 10, + totalTokens: 1010, + }, + }); + expect(usage).toMatchObject({ + input_total: 1000, + input_cached: 600, + input_cache_write: 100, + input_uncached: 300, + }); + }); + + it("never lets a cached count larger than input go negative", () => { + const usage = normalizeUsage({ + harness: "codex", + raw: { inputTokens: 100, cachedInputTokens: 150, outputTokens: 1, totalTokens: 101 }, + }); + expect(usage).toMatchObject({ input_total: 100, input_cached: 100, input_uncached: 0 }); + }); + + it("adds Anthropic cache reads and writes on top of input_tokens (claude_code)", () => { + const usage = normalizeUsage({ + harness: "claude_code", + raw: { + inputTokens: 40, + cachedInputTokens: 127_000, + cacheCreationInputTokens: 3_000, + outputTokens: 900, + totalTokens: 130_940, + }, + }); + expect(usage).toEqual({ + input_total: 130_040, + input_cached: 127_000, + input_cache_write: 3_000, + input_uncached: 40, + output: 900, + reasoning: 0, + reasoning_in_output: true, + convention: "anthropic_cache_separate", + }); + }); + + it("keeps claude_cua cache reads and writes separate until normalization", () => { + const usage = normalizeUsage({ + harness: "claude_cua", + raw: { + inputTokens: 12, + cachedInputTokens: 30, + cacheCreationInputTokens: 8, + outputTokens: 5, + totalTokens: 55, + }, + }); + expect(usage).toMatchObject({ + convention: "anthropic_cache_separate", + input_total: 50, + input_uncached: 12, + input_cached: 30, + input_cache_write: 8, + }); + }); + + it("treats pi input as the uncached remainder (input_tokens=40 with 127k cached observed)", () => { + const usage = normalizeUsage({ + harness: "pi", + raw: { + inputTokens: 40, + cachedInputTokens: 127_000, + cacheCreationInputTokens: 0, + outputTokens: 500, + reasoningOutputTokens: 120, + totalTokens: 127_540, + }, + }); + expect(usage).toMatchObject({ + input_total: 127_040, + input_cached: 127_000, + input_uncached: 40, + output: 500, + reasoning: 120, + convention: "uncached_only", + }); + }); + + it("marks cursor usage as unreported instead of zero", () => { + const usage = normalizeUsage({ + harness: "cursor", + raw: { inputTokens: 0, outputTokens: 0, totalTokens: 0, reported: false }, + }); + expect(usage.convention).toBe("unreported"); + expect(formatNormalizedUsage(usage)).toBe("in=? out=? (usage unreported)"); + }); + + it("treats reported:false as unreported even for a harness that normally reports usage", () => { + const usage = normalizeUsage({ + harness: "codex", + raw: { inputTokens: 0, outputTokens: 0, totalTokens: 0, reported: false }, + }); + expect(usage.convention).toBe("unreported"); + expect( + normalizeUsage({ + harness: "codex", + raw: { inputTokens: 5, outputTokens: 1, totalTokens: 6, reported: true }, + }).convention, + ).toBe("openai_cached_subset"); + }); + + it("falls back to the subset convention for unregistered harnesses", () => { + expect(usageConventionFor("brand_new")).toBe("openai_cached_subset"); + }); + + it("formats the trace token summary from the normalized buckets", () => { + const usage = normalizeUsage({ + harness: "claude_code", + raw: { inputTokens: 10, cachedInputTokens: 90, outputTokens: 5, totalTokens: 105 }, + }); + expect(formatNormalizedUsage(usage)).toBe("in=100 (cached 90) out=5"); + }); +}); diff --git a/packages/evals/tests/initStagehand.test.ts b/packages/evals/tests/initStagehand.test.ts new file mode 100644 index 000000000..a6f291706 --- /dev/null +++ b/packages/evals/tests/initStagehand.test.ts @@ -0,0 +1,127 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { cleanupActiveRunResources } from "../framework/activeRunCleanup.js"; +import { EVAL_SYSTEM_PROMPT } from "../framework/evalSystemPrompt.js"; +import { initStagehand } from "../initStagehand.js"; +import type { EvalLogger } from "../logger.js"; + +const mocks = vi.hoisted(() => ({ + create: vi.fn(), + connectBrowserbase: vi.fn(), + launchLocal: vi.fn(), + launchRemote: vi.fn(), + resolveKey: vi.fn(), +})); + +vi.mock("@browserbasehq/stagehand", () => ({ + Stagehand: { create: mocks.create }, + browserbase: { connect: mocks.connectBrowserbase }, + localBrowser: { launch: mocks.launchLocal }, +})); +vi.mock("../core/targets/browserbase.js", () => ({ + launchRunnerProvidedBrowserbaseChrome: mocks.launchRemote, +})); +vi.mock("../tui/welcomeStatus.js", () => ({ resolveKey: mocks.resolveKey })); + +describe("native Stagehand evaluation initialization", () => { + const page = { id: "test-page" }; + const browser = { close: vi.fn() }; + const stagehand = { + close: vi.fn(), + browser: { context: { activePage: vi.fn() } }, + }; + const releaseSession = vi.fn(); + const logger = { log: vi.fn() } as unknown as EvalLogger; + + beforeEach(() => { + vi.resetAllMocks(); + mocks.resolveKey.mockImplementation((name: string) => ({ + value: + name === "OPENAI_API_KEY" + ? "test-model-key" + : name === "BROWSERBASE_API_KEY" + ? "test-browser-key" + : "", + source: "process-env", + })); + mocks.launchLocal.mockResolvedValue(browser); + mocks.connectBrowserbase.mockResolvedValue(browser); + mocks.launchRemote.mockResolvedValue({ + sessionId: "test-session", + sessionUrl: "https://browserbase.test/sessions/test-session", + debugUrl: "https://browserbase.test/debug/test-session", + cleanup: releaseSession, + }); + mocks.create.mockResolvedValue(stagehand); + stagehand.browser.context.activePage.mockResolvedValue(page); + stagehand.close.mockResolvedValue(undefined); + browser.close.mockResolvedValue(undefined); + releaseSession.mockResolvedValue(undefined); + }); + + afterEach(async () => { + await cleanupActiveRunResources(); + }); + + it.each(["LOCAL", "BROWSERBASE"] as const)( + "passes the shared system prompt through %s initialization and preserves cleanup", + async (environment) => { + const result = await initStagehand({ + logger, + modelName: "openai/gpt-6-astra", + environment, + }); + + expect(mocks.create).toHaveBeenCalledExactlyOnceWith({ + browser, + selfHeal: true, + model: { modelName: "openai/gpt-6-astra", apiKey: "test-model-key" }, + systemPrompt: EVAL_SYSTEM_PROMPT, + logging: { onLog: expect.any(Function) }, + }); + expect(result.stagehand).toBe(stagehand); + expect(result.page).toBe(page); + expect(stagehand.browser.context.activePage).toHaveBeenCalledOnce(); + if (environment === "LOCAL") { + expect(mocks.launchLocal).toHaveBeenCalledExactlyOnceWith({ headless: false }); + expect(mocks.launchRemote).not.toHaveBeenCalled(); + expect(mocks.connectBrowserbase).not.toHaveBeenCalled(); + expect(result.sessionUrl).toBe(""); + expect(result.debugUrl).toBe(""); + } else { + expect(mocks.launchLocal).not.toHaveBeenCalled(); + expect(mocks.launchRemote).toHaveBeenCalledOnce(); + expect(mocks.connectBrowserbase).toHaveBeenCalledExactlyOnceWith({ + apiKey: "test-browser-key", + sessionId: "test-session", + }); + expect(result.sessionUrl).toBe("https://browserbase.test/sessions/test-session"); + expect(result.debugUrl).toBe("https://browserbase.test/debug/test-session"); + } + + await result.cleanup(); + await result.cleanup(); + await cleanupActiveRunResources(); + expect(stagehand.close).toHaveBeenCalledOnce(); + expect(browser.close).toHaveBeenCalledOnce(); + expect(releaseSession).toHaveBeenCalledTimes(environment === "BROWSERBASE" ? 1 : 0); + }, + ); + + it("releases the connected browser and remote session when Stagehand creation fails", async () => { + const error = new Error("synthetic initialization failure"); + mocks.create.mockRejectedValue(error); + + await expect( + initStagehand({ logger, modelName: "openai/gpt-6-astra", environment: "BROWSERBASE" }), + ).rejects.toBe(error); + + expect(mocks.create).toHaveBeenCalledWith( + expect.objectContaining({ systemPrompt: EVAL_SYSTEM_PROMPT }), + ); + await cleanupActiveRunResources(); + expect(browser.close).toHaveBeenCalledOnce(); + expect(releaseSession).toHaveBeenCalledOnce(); + expect(stagehand.close).not.toHaveBeenCalled(); + expect(stagehand.browser.context.activePage).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/evals/tests/scripts/updatePricing.test.ts b/packages/evals/tests/scripts/updatePricing.test.ts new file mode 100644 index 000000000..63600633a --- /dev/null +++ b/packages/evals/tests/scripts/updatePricing.test.ts @@ -0,0 +1,110 @@ +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { EvalsError } from "../../errors.js"; +import { updatePricing } from "../../scripts/update-pricing.js"; + +let directory: string; +let target: string; +const original = JSON.stringify({ + as_of: "2026-01-01", + models: { + "openai/example": { + input_per_m: 1, + cached_input_per_m: 0.1, + output_per_m: 4, + source: "fixture", + }, + "anthropic/pending": { + input_per_m: null, + cached_input_per_m: null, + output_per_m: null, + source: "needs owner input", + }, + }, +}); + +beforeEach(async () => { + directory = await mkdtemp(path.join(tmpdir(), "stagehand-pricing-test-")); + target = path.join(directory, "pricing.json"); + await writeFile(target, original); + vi.stubEnv("EVAL_PRICING_SOURCE", "gateway"); + vi.spyOn(console, "log").mockImplementation(() => {}); + vi.spyOn(console, "warn").mockImplementation(() => {}); +}); + +afterEach(async () => { + vi.unstubAllGlobals(); + vi.unstubAllEnvs(); + vi.restoreAllMocks(); + await rm(directory, { recursive: true, force: true }); +}); + +describe("pricing refresh", () => { + it.each([null, {}, { data: [] }, { data: [null, { id: "broken" }] }])( + "preserves the existing file when HTTP200 has an unusable catalog: %j", + async (body) => { + vi.stubGlobal( + "fetch", + vi.fn(async () => Response.json(body)), + ); + await expect(updatePricing(target)).rejects.toBeInstanceOf(EvalsError); + expect(await readFile(target, "utf8")).toBe(original); + }, + ); + + it("does not unprice every model when a catalog has no matching IDs", async () => { + vi.stubGlobal( + "fetch", + vi.fn(async () => + Response.json({ + data: [{ id: "unrelated/model", pricing: { input: "0.000001", output: "0.000004" } }], + }), + ), + ); + await expect(updatePricing(target)).rejects.toThrow(/currently priced models/u); + expect(await readFile(target, "utf8")).toBe(original); + }); + + it("keeps HTTP failures typed and excludes response secrets", async () => { + vi.stubGlobal( + "fetch", + vi.fn(async () => new Response("credential=do-not-emit", { status: 503 })), + ); + const failure = updatePricing(target); + await expect(failure).rejects.toBeInstanceOf(EvalsError); + await expect(failure).rejects.toThrow("gateway models: HTTP 503"); + await expect(failure).rejects.not.toThrow("do-not-emit"); + expect(await readFile(target, "utf8")).toBe(original); + }); + + it("falls back from an unusable gateway and retains observed cache-write rates for owner input", async () => { + vi.stubEnv("EVAL_PRICING_SOURCE", ""); + const fetchMock = vi + .fn() + .mockResolvedValueOnce(Response.json({ data: [] })) + .mockResolvedValueOnce( + Response.json({ + data: [ + { id: "openai/example", pricing: { prompt: "0.000002", completion: "0.000008" } }, + { + id: "anthropic/pending", + pricing: { + prompt: "0.000003", + completion: "0.000015", + input_cache_write: "0.00000375", + }, + }, + ], + }), + ); + vi.stubGlobal("fetch", fetchMock); + await updatePricing(target); + const updated = JSON.parse(await readFile(target, "utf8")); + expect(fetchMock).toHaveBeenCalledTimes(2); + expect(updated.models["openai/example"].input_per_m).toBe(2); + expect(updated.models["anthropic/pending"]).toMatchObject({ input_per_m: null }); + expect(updated.models["anthropic/pending"].note).toContain("cache_write=3.75"); + }); +}); diff --git a/packages/evals/tests/tui/run.test.ts b/packages/evals/tests/tui/run.test.ts index 8b5494f8d..95e913509 100644 --- a/packages/evals/tests/tui/run.test.ts +++ b/packages/evals/tests/tui/run.test.ts @@ -1,5 +1,6 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import type { DiscoveredTask, TaskRegistry } from "../../framework/types.js"; +import type { RunEvalsResult } from "../../framework/runner.js"; import { canExecuteBenchHarness, deriveCategoryFilter, @@ -8,15 +9,18 @@ import { import { formatBenchHarnessFlags, listBenchHarnessesForTaskKind, + listBenchHarnessesForToolSurface, registerBenchHarness, } from "../../framework/benchHarness.js"; const runEvalsMock = vi.hoisted(() => - vi.fn(async () => ({ - experimentName: "test-experiment", - summary: { passed: 0, failed: 0, total: 0 }, - results: [], - })), + vi.fn( + async (): Promise => ({ + experimentName: "test-experiment", + summary: { passed: 0, failed: 0, total: 0 }, + results: [], + }), + ), ); vi.mock("../../framework/runner.js", () => ({ @@ -209,7 +213,9 @@ describe("deriveCategoryFilter", () => { }, registry, ), - ).rejects.toThrow(formatBenchHarnessFlags(listBenchHarnessesForTaskKind("suite"))); + ).rejects.toThrow( + formatBenchHarnessFlags(listBenchHarnessesForToolSurface("stagehand_facade")), + ); }); it("prints claude_code dry-run matrices without stagehand agent modes", async () => { @@ -253,24 +259,24 @@ describe("deriveCategoryFilter", () => { dataset: "webvoyager", model: "anthropic/claude-sonnet-4-20250514", harness: "claude_code", - toolSurface: "browse_cli", + toolSurface: "stagehand_facade", startupProfile: "tool_create_browserbase", - toolCommand: "browse", - browseCliVersion: expect.any(String), - browseCliEntrypoint: expect.stringMatching(/browse[/\\]bin[/\\]run\.js$/u), + toolCommand: null, + browseCliVersion: null, + browseCliEntrypoint: null, harnessConfig: { harness: "claude_code", model: "anthropic/claude-sonnet-4-20250514", environment: "BROWSERBASE", useApi: false, - toolSurface: "browse_cli", + toolSurface: "stagehand_facade", startupProfile: "tool_create_browserbase", dataset: "webvoyager", }, }); }); - it("prints codex dry-run matrices with browse_cli metadata", async () => { + it("prints codex dry-run matrices with facade metadata", async () => { const registry = makeRegistry([ makeTask({ name: "agent/webvoyager", @@ -311,17 +317,17 @@ describe("deriveCategoryFilter", () => { dataset: "webvoyager", model: "openai/gpt-5.4-mini", harness: "codex", - toolSurface: "browse_cli", + toolSurface: "stagehand_facade", startupProfile: "tool_create_browserbase", - toolCommand: "browse", - browseCliVersion: expect.any(String), - browseCliEntrypoint: expect.stringMatching(/browse[/\\]bin[/\\]run\.js$/u), + toolCommand: null, + browseCliVersion: null, + browseCliEntrypoint: null, harnessConfig: { harness: "codex", model: "openai/gpt-5.4-mini", environment: "BROWSERBASE", useApi: false, - toolSurface: "browse_cli", + toolSurface: "stagehand_facade", startupProfile: "tool_create_browserbase", dataset: "webvoyager", }, @@ -456,6 +462,63 @@ describe("deriveCategoryFilter", () => { expect(output).toContain("Env: BROWSERBASE Harness: stagehand Concurrency: 25"); expect(runEvalsMock).toHaveBeenCalledOnce(); }); + + it("fails a gated batch when a graded pass has zero facade tool calls", async () => { + const previousExitCode = process.exitCode; + const previousLimit = process.env.EVAL_MAX_UNVERIFIABLE_CRITERIA; + const registry = makeRegistry([makeTask()]); + vi.spyOn(console, "log").mockImplementation(() => {}); + const error = vi.spyOn(console, "error").mockImplementation(() => {}); + runEvalsMock.mockResolvedValueOnce({ + experimentName: "test-experiment", + summary: { passed: 1, failed: 0, total: 1 }, + results: [ + { + name: "dropdown", + input: { name: "dropdown", modelName: "openai/gpt-4.1-mini" }, + output: { + _success: true, + criterionCount: 1, + evidenceInsufficient: [], + metrics: { facade_tool_calls: { value: 0 } }, + }, + score: 1, + }, + ], + }); + process.exitCode = undefined; + try { + await runCommand( + { + target: "act", + normalizedTarget: "act", + trials: 1, + concurrency: 1, + environment: "LOCAL", + model: "openai/gpt-4.1-mini", + useApi: false, + harness: "stagehand", + envOverrides: { EVAL_MAX_UNVERIFIABLE_CRITERIA: "0" }, + dryRun: false, + preview: false, + successMode: "outcome", + verbose: false, + }, + registry, + ); + + expect(runEvalsMock).toHaveBeenCalledOnce(); + expect(process.exitCode).toBe(1); + expect(error.mock.calls.flat().join("\n")).toMatch( + /passes without (?:any browser tool call|browser use)/, + ); + expect(process.env.EVAL_MAX_UNVERIFIABLE_CRITERIA).toBe(previousLimit); + } finally { + process.exitCode = previousExitCode; + if (previousLimit === undefined) delete process.env.EVAL_MAX_UNVERIFIABLE_CRITERIA; + else process.env.EVAL_MAX_UNVERIFIABLE_CRITERIA = previousLimit; + } + }); }); describe("buildCombinations (preview column-pruning)", () => { diff --git a/packages/evals/tui/commands/run.ts b/packages/evals/tui/commands/run.ts index d4a2e9f22..1cc32f138 100644 --- a/packages/evals/tui/commands/run.ts +++ b/packages/evals/tui/commands/run.ts @@ -23,8 +23,8 @@ import type { Harness } from "../../framework/benchTypes.js"; import { formatBenchHarnessFlags, isExecutableBenchHarness } from "../../framework/benchHarness.js"; import { armsOverLimit, - armsWithUngradedRuns, armsWithPassesWithoutBrowserUse, + armsWithUngradedRuns, resolveUnverifiableCriteriaLimit, summarizeArmVerifiability, } from "../../framework/verifierGate.js"; diff --git a/packages/evals/types/evals.ts b/packages/evals/types/evals.ts index 9472f0eb1..52b9344f2 100644 --- a/packages/evals/types/evals.ts +++ b/packages/evals/types/evals.ts @@ -76,6 +76,7 @@ export interface Testcase extends EvalCase< api?: boolean; provider?: string; toolSurface?: string; + promptVariant?: string; startupProfile?: string; toolCommand?: string; browseCliVersion?: string; @@ -115,6 +116,7 @@ export interface Testcase extends EvalCase< api?: boolean; provider?: string; toolSurface?: string; + promptVariant?: string; startupProfile?: string; toolCommand?: string; browseCliVersion?: string;