From bddda59357429e170ae801d34d376ebb002877ad Mon Sep 17 00:00:00 2001 From: miguel Date: Mon, 7 Sep 2026 16:49:20 -0700 Subject: [PATCH 1/4] Keep verifier outcomes, uncertainty, and execution failures auditable --- packages/evals/docs/verifier-gates.md | 27 + .../framework/harnesses/persistTrajectory.ts | 16 +- .../framework/harnesses/trajectoryAdapter.ts | 20 + packages/evals/framework/verifierAdapter.ts | 242 ++++++- packages/evals/framework/verifierGate.ts | 28 +- packages/evals/framework/verifierGates.ts | 487 +++++++++++++ packages/evals/framework/verifierTrace.ts | 53 ++ packages/evals/logger.ts | 10 +- .../framework/gradeExternalTrajectory.test.ts | 241 ++++++- .../framework/harnessObservations.test.ts | 57 +- .../tests/framework/persistTrajectory.test.ts | 26 + .../tests/framework/verifierGates.test.ts | 668 ++++++++++++++++++ packages/evals/tests/logger.test.ts | 12 + packages/evals/tests/tui/verify.test.ts | 36 + packages/evals/tui/commands/verify.ts | 69 +- 15 files changed, 1949 insertions(+), 43 deletions(-) create mode 100644 packages/evals/docs/verifier-gates.md create mode 100644 packages/evals/framework/verifierGates.ts create mode 100644 packages/evals/framework/verifierTrace.ts create mode 100644 packages/evals/tests/framework/verifierGates.test.ts create mode 100644 packages/evals/tests/tui/verify.test.ts diff --git a/packages/evals/docs/verifier-gates.md b/packages/evals/docs/verifier-gates.md new file mode 100644 index 0000000000..071a3ffacf --- /dev/null +++ b/packages/evals/docs/verifier-gates.md @@ -0,0 +1,27 @@ +# Verifier evidence gates + +External harnesses use the existing V3 verifier with the task's precomputed rubric when available. A requested verification that errors or returns a verifier-uncertainty sentinel fails closed: `_success` is false, `verifierError` explains the failure, and `agentReportedSuccess` preserves the original self-report. Such rows are ungraded and must not be presented as verified benchmark outcomes. + +The raw judge verdict is retained under `judge` in `scores/result.json`. The top-level result, `task_data.json`, and the run row carry the same adjusted outcome. `scores/gates.json` records diagnostics when trajectory persistence is enabled. + +## Outcome + +A judge pass is rejected when the final answer is empty or, if a mounted-tool matcher is available, the trajectory has no browser tool calls. Grounding checks are advisory by default. `EVAL_REQUIRE_GROUNDING=1` additionally rejects an answer whose checked numeric findings all lack matching non-search-page observations. The check is a text heuristic: images, paraphrases and valid snippet sources can escape its matching, so it is not a replacement for rubric verification. + +Execution state remains separate. `harnessStatus`, `harnessStopReason` and `terminationReason` distinguish completion, budget exhaustion, abort, SDK error and browser loss. A supported task completion can still pass after a disconnect; execution error alone does not erase earlier evidence. An unfinished task must fail its rubric. + +## Process + +`processScoreStrict` recomputes the weighted process score with explicit `evidenceInsufficient` criteria earning zero while retaining their maximum points in the denominator. `processScore` uses this score; `processScoreLenient` preserves the judge's aggregate. Not-applicable criteria are excluded. Without a criterion breakdown, the judge's aggregate is retained and `scoringIncomplete` flags a short result against the rubric. + +Blocker wording is recorded as `blockerMentioned` on criterion diagnostics. It never changes points by itself: permitted fallback and stop-boundary explanations can correctly mention a blocker. The rubric and observed evidence determine whether the requirement was satisfied. This replaces the campaign's overbroad blocker substring heuristic. + +## Reporting + +- `facade_tool_calls`, `facade_tool_call_failures` and `facade_tool_calls_after_session_lost` distinguish attempted browser work from repeated terminal failures. +- Agent, evidence-capture and verifier wall times are reported separately. +- Normalized usage records the SDK's cache convention. Missing usage is unavailable; historical Cursor CLI records remain unreported. +- `cost_source=reported` means the harness reported dollars. `computed` is an estimate using the dated catalog snapshot in `pricing/pricing.json`; it is not an invoice. Unknown or subscription costs are unavailable, never inferred as zero. +- `harnessImplementation` records adapter and SDK versions when supplied. Its absence in older records means unknown implementation; historical labels are preserved. + +Use `VERIFIER_PERSIST_TRAJECTORIES=1` for reviewable evidence. HardBench's compatibility gate rejects verifier errors, uncertainty sentinels, missing criteria and self-report fallbacks before accepting a result. Offline transport checks establish integration compatibility; live rubric accuracy still requires the separately recorded live fixtures. diff --git a/packages/evals/framework/harnesses/persistTrajectory.ts b/packages/evals/framework/harnesses/persistTrajectory.ts index 0eb9353ab3..73a9336ba6 100644 --- a/packages/evals/framework/harnesses/persistTrajectory.ts +++ b/packages/evals/framework/harnesses/persistTrajectory.ts @@ -7,10 +7,11 @@ import { resolveTrajectoryRoot, writeTrajectoryMetadata, } from "../trajectoryGroup.js"; -import type { EvaluationResult, TaskSpec, Trajectory } from "stagehand-v3"; +import type { EvaluationResult, TaskSpec } from "stagehand-v3"; +import type { HarnessTrajectory } from "./trajectoryAdapter.js"; export interface PersistAdapterTrajectoryOptions { - trajectory: Trajectory; + trajectory: HarnessTrajectory; taskSpec: TaskSpec; /** EvaluationResult from V3Evaluator.verify(). Written to scores/result.json. */ evaluationResult?: EvaluationResult; @@ -73,6 +74,16 @@ export async function persistAdapterTrajectory( runDir: path.basename(directory), attempt, status: opts.trajectory.status, + ...(opts.trajectory.cost_pricing && { cost_pricing: opts.trajectory.cost_pricing }), + ...(opts.trajectory.terminationReason && { + terminationReason: opts.trajectory.terminationReason, + }), + ...(opts.trajectory.harnessImplementation && { + harnessImplementation: opts.trajectory.harnessImplementation, + }), + ...(opts.trajectory.harnessConfiguration && { + harnessConfiguration: opts.trajectory.harnessConfiguration, + }), }); if (opts.evaluationResult) { @@ -86,6 +97,7 @@ export async function persistAdapterTrajectory( { task: opts.trajectory.task, status: opts.trajectory.status, + ...(opts.trajectory.cost_pricing && { cost_pricing: opts.trajectory.cost_pricing }), finalAnswer: opts.trajectory.finalAnswer ?? null, result: opts.evaluationResult, }, diff --git a/packages/evals/framework/harnesses/trajectoryAdapter.ts b/packages/evals/framework/harnesses/trajectoryAdapter.ts index 2d43cf6d50..4dd86f2bab 100644 --- a/packages/evals/framework/harnesses/trajectoryAdapter.ts +++ b/packages/evals/framework/harnesses/trajectoryAdapter.ts @@ -7,6 +7,26 @@ import type { TrajectoryStep, } from "stagehand-v3"; +/** + * Why a run stopped. `status` on the Trajectory only says whether it ended + * cleanly; this says what ended it, so a budget-exhausted run can be told + * apart from a crash offline. + */ +export type TerminationReason = + | "completed" + | "step_budget" + | "browser_session_lost" + | "sdk_error" + | "aborted"; + +/** A Trajectory as external harnesses persist it. */ +export type HarnessTrajectory = Trajectory & { + terminationReason?: TerminationReason; + harnessImplementation?: { name: string; version: number; sdkVersion?: string }; + harnessConfiguration?: Record; + cost_pricing?: { as_of: string; model: string; source: string }; +}; + /** * Pure converter from a harness-specific result to a verifier Trajectory. * Implementations must be deterministic (no I/O, no mutation of input). diff --git a/packages/evals/framework/verifierAdapter.ts b/packages/evals/framework/verifierAdapter.ts index d7f1a3d8cf..91a6634448 100644 --- a/packages/evals/framework/verifierAdapter.ts +++ b/packages/evals/framework/verifierAdapter.ts @@ -10,29 +10,73 @@ import { type V3, } from "stagehand-v3"; +import fs from "node:fs/promises"; +import path from "node:path"; +import { sanitizeErrorMessage } from "@browserbasehq/stagehand-integrations/harness"; + import type { EvalLogger } from "../logger.js"; import { tracedSpan } from "./braintrust.js"; import { persistAdapterTrajectory } from "./harnesses/persistTrajectory.js"; +import { + selectVerifierTraceLines, + verifierTraceEnabled, + writeVerifierTrace, +} from "./verifierTrace.js"; +import type { HarnessTrajectory } from "./harnesses/trajectoryAdapter.js"; import { RubricCache } from "./rubricCache.js"; import type { TaskResult } from "./types.js"; +import { applyVerdictGates, resolveRequireGrounding, type VerdictGates } from "./verifierGates.js"; + +/** + * What scores/result.json holds: the judge's EvaluationResult shape with the + * gated verdict at the top level, so a reader of result.json alone sees the + * same outcome as the Braintrust row. The judge's untouched verdict is kept + * under `judge` (and as `judgeOutcomeSuccess` / `processScoreLenient`). + */ +export interface PersistedEvaluationResult extends EvaluationResult { + judgeOutcomeSuccess: boolean; + outcomeGates: VerdictGates["outcomeGates"]; + processScoreStrict: number | undefined; + processScoreLenient: number | undefined; + judge: EvaluationResult; +} + +export function buildPersistedEvaluationResult( + evaluation: EvaluationResult, + gates: VerdictGates, +): PersistedEvaluationResult { + return { + ...evaluation, + outcomeSuccess: gates.outcomeSuccess, + processScore: gates.processScore, + ...(gates.perCriterion && { perCriterion: gates.perCriterion }), + judgeOutcomeSuccess: gates.judgeOutcomeSuccess, + outcomeGates: gates.outcomeGates, + processScoreStrict: gates.processScoreStrict, + processScoreLenient: gates.processScoreLenient, + judge: evaluation, + }; +} const VERIFIER_MODEL_ENV = "EVAL_VERIFIER_MODEL"; const KEYLESS_VERIFIER_PROVIDERS = new Set(["bedrock", "ollama"]); +/** Campaign-validated default; callers can pin the judge independently. */ +export const DEFAULT_VERIFIER_MODEL = "google/gemini-3.5-flash"; /** - * Build the shared rubric verifier. By default V3Evaluator keeps its existing - * model selection; EVAL_VERIFIER_MODEL makes the verifier independently - * selectable for external harnesses and normal Stagehand runs alike. + * Build the shared rubric verifier. EVAL_VERIFIER_MODEL makes the verifier + * independently selectable for external harnesses and normal Stagehand runs + * alike; otherwise DEFAULT_VERIFIER_MODEL applies. */ export function createVerifierEvaluator(v3: V3): V3Evaluator { - const modelName = process.env[VERIFIER_MODEL_ENV]?.trim(); - if (!modelName) { - return new V3Evaluator(v3, { backend: "verifier" }); - } + const explicitModel = process.env[VERIFIER_MODEL_ENV]?.trim(); + const modelName = explicitModel || DEFAULT_VERIFIER_MODEL; const provider = modelName.includes("/") ? modelName.slice(0, modelName.indexOf("/")) : undefined; const apiKey = loadApiKeyFromEnv(provider, () => {}); - if (!apiKey && !KEYLESS_VERIFIER_PROVIDERS.has(provider ?? "")) { + // Only an explicit override fails loudly on a missing key; the default lets + // V3Evaluator resolve credentials itself (tests and keyless environments). + if (explicitModel && !apiKey && !KEYLESS_VERIFIER_PROVIDERS.has(provider ?? "")) { throw new Error( `${VERIFIER_MODEL_ENV} is set to "${modelName}", but no API key was found for provider "${provider ?? "unknown"}".`, ); @@ -192,7 +236,7 @@ export interface ExternalHarnessVerifierConfig { export interface GradeExternalTrajectoryOptions { /** Builds the harness-specific Trajectory; runs inside the guarded block. */ - buildTrajectory: () => Trajectory; + buildTrajectory: () => HarnessTrajectory; verifier: ExternalHarnessVerifierConfig; /** The agent's self-reported result to fold the verdict into. */ baseResult: TaskResult; @@ -201,13 +245,18 @@ export interface GradeExternalTrajectoryOptions { /** Logger category ("claude_code" | "codex"). */ category: string; logger: EvalLogger; + /** + * Matcher for mounted-browser (facade) tool names. When present, a judge + * pass with zero facade steps is gated (`no_browser_use`). + */ + isFacadeTool?: (name: string) => boolean; } /** * Grade an external-harness run with the rubric verifier and fold the verdict * into the TaskResult. Never throws: on any failure in the verifier path the - * self-reported result is returned with `verifierError` set, so downstream - * consumers can tell an ungraded run apart from a graded one. + * result fails closed with `verifierError` set and the agent report preserved + * separately, so downstream consumers can distinguish ungraded runs. */ export async function gradeExternalTrajectory({ buildTrajectory, @@ -216,9 +265,14 @@ export async function gradeExternalTrajectory({ errorMessage, category, logger, + isFacadeTool, }: GradeExternalTrajectoryOptions): Promise { + let capturedTrajectory: HarnessTrajectory | undefined; + let rawEvaluation: EvaluationResult | undefined; + let savedDirectory: string | undefined; try { const trajectory = buildTrajectory(); + capturedTrajectory = trajectory; const evaluator = createVerifierEvaluator(verifier.v3); // Hydrate rubric — use precomputed if present, otherwise cache-or-generate. @@ -230,42 +284,127 @@ export async function gradeExternalTrajectory({ ...verifier.taskSpec, precomputedRubric: rubric, }; - const hydratedTrajectory = { ...trajectory, task: hydratedSpec }; + const hydratedTrajectory: HarnessTrajectory = { ...trajectory, task: hydratedSpec }; + capturedTrajectory = hydratedTrajectory; + const traceOn = verifierTraceEnabled(); + const logCountBefore = traceOn ? logger.getLogs({ maxLevel: 2 }).length : 0; const evaluationResult = await verifyTraced(evaluator, hydratedTrajectory, { taskId: hydratedSpec.id, dataset: verifier.dataset, }); + rawEvaluation = evaluationResult; + if (evaluationResult.findings?.some((finding) => finding.category === "verifier_uncertainty")) { + throw new Error( + "Verifier returned an uncertainty result; no trustworthy grade was produced.", + ); + } + const traceLines = traceOn + ? selectVerifierTraceLines(logger.getLogs({ maxLevel: 2 }), logCountBefore) + : []; + // The judge's verdict is not the final word: deterministic gates fold in + // what the trajectory itself proves (an answer exists, the run finished, + // the browser was used, the numbers came from the target site) and a + // strict process score that does not credit blocker-walled criteria. See + // verifierGates.ts for why each exists. + const gates = applyVerdictGates({ + evaluation: evaluationResult, + trajectory: hydratedTrajectory, + isFacadeTool, + requireGrounding: resolveRequireGrounding( + verifier.dataset, + Boolean(verifier.taskSpec.precomputedRubric), + ), + rubricItemCount: rubric.items.length, + }); const successMode = verifier.successMode ?? process.env.EVAL_SUCCESS_MODE; - const verifiedSuccess = evaluationResultToSuccess(evaluationResult, successMode); + const verifiedSuccess = evaluationResultToSuccess( + { + ...evaluationResult, + outcomeSuccess: gates.outcomeSuccess, + processScore: gates.processScore, + }, + successMode, + ); - const { directory: trajectoryDir } = await persistAdapterTrajectory({ + const { directory: trajectoryDir, persisted } = await persistAdapterTrajectory({ trajectory: hydratedTrajectory, taskSpec: hydratedSpec, - evaluationResult, + evaluationResult: buildPersistedEvaluationResult(evaluationResult, gates), outputRoot: verifier.trajectoryRoot, runId: verifier.runId, }); + savedDirectory = trajectoryDir; + if (persisted) await writeGatesFile(trajectoryDir, gates); + if (persisted && traceLines.length) { + const file = await writeVerifierTrace(trajectoryDir, traceLines); + if (file) logger.log({ category, message: `verifier trace: ${file}`, level: 1 }); + } + const gateSuffix = gates.outcomeGates.length ? ` gated=${gates.outcomeGates.join(",")}` : ""; logger.log({ category, - message: `result: outcome=${evaluationResult.outcomeSuccess} process=${formatProcessScore(evaluationResult.processScore)} steps=${hydratedTrajectory.steps.length}`, + message: `result: outcome=${gates.outcomeSuccess} (judge=${gates.judgeOutcomeSuccess}${gateSuffix}) process=${formatProcessScore(gates.processScore)} (lenient=${formatProcessScore(gates.processScoreLenient)}) steps=${hydratedTrajectory.steps.length}`, level: 1, }); return { ...baseResult, _success: verifiedSuccess, - error: verifiedSuccess ? undefined : (baseResult.error ?? errorMessage), - outcomeSuccess: evaluationResult.outcomeSuccess, - processScore: evaluationResult.processScore, + error: verifiedSuccess + ? undefined + : gates.outcomeGates.length > 0 && gates.judgeOutcomeSuccess + ? // The judge passed this row; a deterministic gate flipped it. Say + // so where the row error is read, instead of echoing the agent's + // (often confident) self-report. + `${describeOutcomeGates(gates)} (judge passed; agent said: ${clipError(String(baseResult.error ?? errorMessage))})` + : (baseResult.error ?? errorMessage), + outcomeSuccess: gates.outcomeSuccess, + judgeOutcomeSuccess: gates.judgeOutcomeSuccess, + outcomeGates: gates.outcomeGates, + processScore: gates.processScore, + processScoreStrict: gates.processScoreStrict, + processScoreLenient: gates.processScoreLenient, + perCriterion: gates.perCriterion, evidenceInsufficient: evaluationResult.evidenceInsufficient, + ...(gates.grounding && { grounding: gates.grounding }), + scoringIncomplete: gates.scoringIncomplete, criterionCount: rubric.items.length, stepCount: hydratedTrajectory.steps.length, trajectoryDir, + metrics: { + ...(asRecord(baseResult.metrics) ?? {}), + ...gateMetrics(gates), + }, }; } catch (verifyError) { - const message = stringifyVerifierError(verifyError); + const message = sanitizeErrorMessage(stringifyVerifierError(verifyError)); + // Failed verification still needs the captured browser evidence and raw + // judge response for diagnosis. This path never accepts the result as a grade. + if (capturedTrajectory && !savedDirectory) { + try { + const saved = await persistAdapterTrajectory({ + trajectory: capturedTrajectory, + taskSpec: capturedTrajectory.task ?? verifier.taskSpec, + ...(rawEvaluation && { evaluationResult: rawEvaluation }), + outputRoot: verifier.trajectoryRoot, + runId: verifier.runId, + }); + savedDirectory = saved.directory; + if (saved.persisted) { + await fs.writeFile( + path.join(saved.directory, "scores", "verifier-error.json"), + JSON.stringify({ verifierError: message, graded: false }, null, 2), + ); + } + } catch (persistenceError) { + logger.warn({ + category, + level: 1, + message: `could not persist failed verification: ${sanitizeErrorMessage(stringifyVerifierError(persistenceError))}`, + }); + } + } logger.warn({ category, message: `verifier integration failed: ${message}`, @@ -274,10 +413,16 @@ export async function gradeExternalTrajectory({ error: { value: message, type: "string" }, }, }); - // Surface the failure on the result — `_success` falls back to the - // agent's self-report, and downstream consumers must be able to tell - // this run apart from one the verifier actually graded. - return { ...baseResult, verifierError: message }; + // A requested verification that failed cannot produce a verified pass. + // Preserve the agent's report separately for diagnosis. + return { + ...baseResult, + _success: false, + agentReportedSuccess: baseResult._success, + verifierError: message, + error: `Verification failed: ${message}`, + ...(savedDirectory && { trajectoryDir: savedDirectory }), + }; } } @@ -285,6 +430,38 @@ function formatProcessScore(score: number | undefined): string { return typeof score === "number" ? score.toFixed(2) : "n/a"; } +/** + * Braintrust-filterable 0/1 metrics for the gates. `answer_grounded` is only + * emitted when the answer had numeric datums to check, so its average is not + * diluted by rows the check skipped. + */ +function gateMetrics(gates: VerdictGates): Record { + const flag = (value: boolean) => ({ count: 1, value: value ? 1 : 0 }); + return { + outcome_gated: flag(gates.outcomeGates.length > 0), + scoring_incomplete: flag(gates.scoringIncomplete), + blocked_criteria: { count: 1, value: gates.blockedCriteria }, + ...(typeof gates.processScoreLenient === "number" && { + process_score_lenient: { count: 1, value: gates.processScoreLenient }, + }), + ...(gates.grounding && { + answer_grounded: flag(!gates.grounding.gatesOutcome), + }), + }; +} + +/** Sidecar next to scores/result.json so audits can diff judge vs gated verdicts. */ +async function writeGatesFile(trajectoryDir: string, gates: VerdictGates): Promise { + try { + await fs.writeFile( + path.join(trajectoryDir, "scores", "gates.json"), + JSON.stringify(gates, null, 2), + ); + } catch { + // Best-effort: the TaskResult already carries the same data. + } +} + /** Always non-empty, so a set `verifierError` is reliably truthy downstream. */ function stringifyVerifierError(value: unknown): string { if (value instanceof Error) return value.message || value.name || "Error"; @@ -342,3 +519,20 @@ export function evaluationResultToSuccess( return outcomeOk && processOk; } } + +const GATE_DESCRIPTIONS: Record = { + no_final_answer: "agent produced no final answer", + trajectory_error: "trajectory ended in error", + no_browser_use: "no browser tool calls", + ungrounded_answer: "answer datums only in search-engine results, never on a target page", +}; + +function describeOutcomeGates(gates: { outcomeGates: string[] }): string { + const parts = gates.outcomeGates.map((g) => GATE_DESCRIPTIONS[g] ?? g); + return `gated: ${gates.outcomeGates.join(",")} — ${parts.join("; ")}`; +} + +function clipError(value: string | undefined): string { + const text = (value ?? "").replace(/\s+/g, " ").trim(); + return text.length > 160 ? `${text.slice(0, 159)}…` : text; +} diff --git a/packages/evals/framework/verifierGate.ts b/packages/evals/framework/verifierGate.ts index a2fd2a469b..e0f4e10a14 100644 --- a/packages/evals/framework/verifierGate.ts +++ b/packages/evals/framework/verifierGate.ts @@ -15,16 +15,31 @@ export interface ArmVerifiability { gradedRuns: number; /** * Verifier-backed runs the verifier failed to grade (`verifierError` set) — - * their `_success` is the agent's self-report, so they must never hide - * inside a gated batch. + * keep these separate from completed grades, including historical rows + * whose `_success` may still contain a self-report. */ ungradedRuns: number; unverifiableCriteria: number; totalCriteria: number; + /** + * Runs marked successful whose `facade_tool_calls` metric is 0 — the agent + * never reached the mounted browser surface, so the pass came from + * somewhere the verifier cannot see (another tool, prior knowledge). + */ + passesWithoutBrowserUse: number; } const GATE_ENV = "EVAL_MAX_UNVERIFIABLE_CRITERIA"; +function readFacadeToolCalls(output: Record): number | undefined { + const metrics = output.metrics; + if (typeof metrics !== "object" || metrics === null) return undefined; + const metric = (metrics as Record).facade_tool_calls; + if (typeof metric !== "object" || metric === null) return undefined; + const value = (metric as { value?: unknown }).value; + return typeof value === "number" ? value : undefined; +} + export function summarizeArmVerifiability( results: Array<{ input: EvalInput; output: Record }>, harness: string, @@ -43,6 +58,7 @@ export function summarizeArmVerifiability( ungradedRuns: 0, unverifiableCriteria: 0, totalCriteria: 0, + passesWithoutBrowserUse: 0, }; if (graded) { arm.gradedRuns += 1; @@ -50,6 +66,9 @@ export function summarizeArmVerifiability( arm.unverifiableCriteria += Array.isArray(output.evidenceInsufficient) ? output.evidenceInsufficient.length : 0; + if (output._success === true && readFacadeToolCalls(output) === 0) { + arm.passesWithoutBrowserUse += 1; + } } else { arm.ungradedRuns += 1; } @@ -78,3 +97,8 @@ export function armsOverLimit(arms: ArmVerifiability[], limit: number): ArmVerif export function armsWithUngradedRuns(arms: ArmVerifiability[]): ArmVerifiability[] { return arms.filter((arm) => arm.ungradedRuns > 0); } + +/** Arms where at least one pass never touched the mounted browser surface. */ +export function armsWithPassesWithoutBrowserUse(arms: ArmVerifiability[]): ArmVerifiability[] { + return arms.filter((arm) => arm.passesWithoutBrowserUse > 0); +} diff --git a/packages/evals/framework/verifierGates.ts b/packages/evals/framework/verifierGates.ts new file mode 100644 index 0000000000..e9337c97b4 --- /dev/null +++ b/packages/evals/framework/verifierGates.ts @@ -0,0 +1,487 @@ +/** + * Deterministic diagnostics and evidence gates over the current V3 verifier. + * Preserve the judge's verdict for auditing. Execution status and blocker + * wording alone cannot establish whether a rubric requirement was completed. + */ +import type { CriterionScore, EvaluationResult, Trajectory, TrajectoryStep } from "stagehand-v3"; + +export type OutcomeGate = "no_final_answer" | "no_browser_use" | "ungrounded_answer"; + +export type GroundingDatumKind = "currency" | "percent" | "time" | "decimal" | "integer" | "entity"; + +export interface GroundingDatum { + /** The token as written in the final answer. */ + text: string; + kind: GroundingDatumKind; + /** Index of the first non-search-engine step whose output contains it. */ + groundedAtStep?: number; + /** True when the datum only ever appeared in search-engine step outputs. */ + onlyInSearchEngine: boolean; + /** Echoed from the task instruction; reported but never gates. */ + fromInstruction?: boolean; +} + +export interface GroundingResult { + checked: GroundingDatum[]; + /** Subset of `checked` that no non-search-engine step output contains. */ + ungrounded: GroundingDatum[]; + /** + * Counts over datums of a kind allowed to gate the outcome (currency / + * percent / time / decimal / integer with 4+ digits). Entity and + * short-integer misses are advisory only. + */ + groundedNumeric: number; + ungroundedNumeric: number; + /** True when the answer has numeric datums and none was verified off-search. */ + gatesOutcome: boolean; +} + +export interface GatedCriterionScore extends CriterionScore { + /** + * Flagged by the judge as lacking evidence. Zeroed in the strict + * process score. + */ + blocked?: boolean; + /** Advisory wording match; a sanctioned fallback or stop boundary may mention a blocker. */ + blockerMentioned?: boolean; +} + +export interface VerdictGates { + /** Gated verdict: judge AND every deterministic gate. */ + outcomeSuccess: boolean; + /** The judge's raw verdict, untouched. */ + judgeOutcomeSuccess: boolean; + /** Gates that flipped a judge pass to a fail. Empty when nothing fired. */ + outcomeGates: OutcomeGate[]; + /** Same as `processScoreStrict`; the score `--success process` reads. */ + processScore: number | undefined; + processScoreStrict: number | undefined; + /** The judge's processScore, untouched. */ + processScoreLenient: number | undefined; + perCriterion: GatedCriterionScore[] | undefined; + /** Count of criteria zeroed in the strict score. */ + blockedCriteria: number; + /** Undefined when grounding was not checked (no final answer / no datums). */ + grounding?: GroundingResult; + /** Rubric has more items than the judge returned criterion scores for. */ + scoringIncomplete: boolean; + rubricItemCount?: number; +} + +export interface ApplyVerdictGatesInput { + evaluation: EvaluationResult; + trajectory: Pick & { + task?: { instruction?: string }; + }; + /** Matcher for mounted-browser (facade) tool names; enables `no_browser_use`. */ + isFacadeTool?: (name: string) => boolean; + /** Whether an ungrounded numeric datum in the answer fails the outcome. */ + requireGrounding: boolean; + /** `task_data.precomputed_rubric.items.length`; enables `scoringIncomplete`. */ + rubricItemCount?: number; +} + +const BLOCKER_EXPLANATION = /uncontrollable|blocker|could not be attempted|blocked/i; + +const SEARCH_ENGINE_HOST = /(^|\.)(google|bing|duckduckgo|yahoo|scribd|reddit)\./i; + +/** Datasets whose rubrics are precomputed and whose answers are factual lookups. */ + +export function applyVerdictGates({ + evaluation, + trajectory, + isFacadeTool, + requireGrounding, + rubricItemCount, +}: ApplyVerdictGatesInput): VerdictGates { + const judgeOutcomeSuccess = evaluation.outcomeSuccess === true; + const finalAnswer = (trajectory.finalAnswer ?? "").trim(); + const outcomeGates: OutcomeGate[] = []; + + if (!finalAnswer) outcomeGates.push("no_final_answer"); + // A disconnect after the required actions does not erase their evidence. + // Execution status is retained separately; the judge must assess completion. + if (isFacadeTool && !trajectory.steps.some((step) => isFacadeTool(step.actionName))) { + outcomeGates.push("no_browser_use"); + } + + const grounding = finalAnswer + ? checkAnswerGrounding(finalAnswer, trajectory.steps, trajectory.task?.instruction ?? "") + : undefined; + if (requireGrounding && grounding?.gatesOutcome) { + outcomeGates.push("ungrounded_answer"); + } + + const { perCriterion, strict, blockedCriteria } = strictProcessScore(evaluation); + + const scoringIncomplete = + typeof rubricItemCount === "number" && + rubricItemCount > 0 && + rubricItemCount > (evaluation.perCriterion?.length ?? 0); + + return { + // Gates only ever flip a pass to a fail; a judge fail stays a fail even + // when no gate fires, so the gate list is only meaningful on a judge pass. + outcomeSuccess: judgeOutcomeSuccess && outcomeGates.length === 0, + judgeOutcomeSuccess, + outcomeGates: judgeOutcomeSuccess ? outcomeGates : [], + processScore: strict, + processScoreStrict: strict, + processScoreLenient: evaluation.processScore, + perCriterion, + blockedCriteria, + ...(grounding && { grounding }), + scoringIncomplete, + ...(rubricItemCount !== undefined && { rubricItemCount }), + }; +} + +/** + * Resolve EVAL_REQUIRE_GROUNDING. Defaults on for datasets whose tasks ship + * precomputed rubrics (factual lookups where a snippet-lifted number is the + * dominant false positive) and off elsewhere. + */ +export function resolveRequireGrounding( + dataset: string, + hasPrecomputedRubric: boolean, + env: NodeJS.ProcessEnv = process.env, +): boolean { + const raw = env.EVAL_REQUIRE_GROUNDING?.trim(); + if (raw === "1" || raw?.toLowerCase() === "true") return true; + if (raw === "0" || raw?.toLowerCase() === "false") return false; + // Advisory by default: a factually correct answer sourced from a search + // snippet still counts as a pass (owner decision, 2026-08-31 — the F1 race + // time answered from Google was ruled a pass). The check still runs and is + // recorded as `grounding` + metric answer_grounded so snippet-sourced passes + // remain filterable; opt into gating with EVAL_REQUIRE_GROUNDING=1. + void dataset; + void hasPrecomputedRubric; + return false; +} + +/** + * Recompute the process score with explicitly evidenceInsufficient-credited + * criteria zeroed. Blocked criteria stay in the denominator: dropping them + * would score a fully bot-walled run 0/0 and, with any fallback, let it pass + * `--success process` again — exactly the leak this exists to close. + * Not-applicable criteria (earnedPoints === null) are excluded, as the judge + * does. + */ +export function strictProcessScore( + evaluation: Pick, +): { + perCriterion: GatedCriterionScore[] | undefined; + strict: number | undefined; + blockedCriteria: number; +} { + const source = evaluation.perCriterion; + // Without a per-criterion breakdown there is nothing to zero; the judge's + // aggregate is the best available strict score. + if (!source) + return { perCriterion: undefined, strict: evaluation.processScore, blockedCriteria: 0 }; + + const insufficient = new Set(evaluation.evidenceInsufficient ?? []); + let earned = 0; + let max = 0; + let blockedCriteria = 0; + const perCriterion = source.map((criterion): GatedCriterionScore => { + const blocked = + criterion.evidenceInsufficient === true || insufficient.has(criterion.criterion); + const blockerMentioned = BLOCKER_EXPLANATION.test(criterion.explanation ?? ""); + // V3 also uses null for a missing judge score, with explicit insufficiency. + // Only genuinely inapplicable criteria leave the denominator. + if (criterion.earnedPoints === null && !blocked) return { ...criterion }; + max += criterion.maxPoints; + if (blocked) blockedCriteria += 1; + else earned += criterion.earnedPoints ?? 0; + return { + ...criterion, + ...(blocked && { blocked: true }), + ...(blockerMentioned && { blockerMentioned: true }), + }; + }); + + return { + perCriterion, + strict: max > 0 ? earned / max : undefined, + blockedCriteria, + }; +} + +// --------------------------------------------------------------------------- +// Grounding +// --------------------------------------------------------------------------- + +const CURRENCY_CODES = "SGD|USD|MYR|EUR|GBP|AUD|CAD|INR|JPY|CNY|HKD|NZD|CHF|THB|IDR|PHP|KRW"; +const CURRENCY_PREFIX = new RegExp( + `(?:(${CURRENCY_CODES}|S\\$|US\\$|A\\$|C\\$|\\$|€|£|¥)\\s?)`, + "i", +); +const TIME_TOKEN = /\b\d{1,2}:\d{2}(?::\d{2})?(?:\.\d{1,3})?\b/g; +const YEAR_TOKEN = /^(?:1[89]|20)\d{2}$/; +const NUMBER_TOKEN = new RegExp( + `${CURRENCY_PREFIX.source}?(\\d[\\d,]*(?:\\.\\d+)?)(?:\\s?(${CURRENCY_CODES}|%|\\$))?`, + "gi", +); +// Two or more Capitalised words, allowing a lowercase joiner in between. +const ENTITY_TOKEN = + /\b[A-Z][\w’'&-]*(?:\s+(?:of|the|and|de|for|&)\s+|\s+)(?:[A-Z][\w’'&-]*)(?:\s+(?:[A-Z][\w’'&-]*))*/g; + +const CURRENCY_ALIASES: Record = { + $: ["$", "usd", "us$"], + usd: ["usd", "us$", "$"], + us$: ["us$", "usd", "$"], + sgd: ["sgd", "s$"], + s$: ["s$", "sgd"], + aud: ["aud", "a$"], + a$: ["a$", "aud"], + cad: ["cad", "c$"], + c$: ["c$", "cad"], + "€": ["€", "eur"], + eur: ["eur", "€"], + "£": ["£", "gbp"], + gbp: ["gbp", "£"], + "¥": ["¥", "jpy", "cny"], + jpy: ["jpy", "¥"], + cny: ["cny", "¥", "rmb"], +}; + +interface ParsedDatum { + text: string; + kind: GroundingDatumKind; + /** Matchers tried against a normalized step text; any hit grounds the datum. */ + patterns: RegExp[]; + /** Whether an ungrounded miss may gate the outcome. */ + gates: boolean; + /** Echoed verbatim from the task instruction — task context, not a finding. */ + fromInstruction?: boolean; +} + +/** + * Lowercase, drop thousands separators, collapse whitespace to one space. Space + * is kept (not removed) so digit boundaries survive: "1:27:02.624 2 Pits" must + * not read as "...6242". + */ +export function normalizeForGrounding(text: string): string { + return text.toLowerCase().replace(/,/g, "").replace(/\s+/g, " ").trim(); +} + +/** Regex source matching `text` with any run of whitespace between its tokens. */ +function spaceTolerant(text: string): string { + return normalizeForGrounding(text).split(" ").map(escapeRegExp).join("\\s*"); +} + +function escapeRegExp(value: string): string { + return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} + +/** + * Extract the datums worth grounding from a final answer. Datums that also + * appear in the task instruction (dates, budgets, quantities the task set) are + * kept for reporting but never gate: the agent did not find them anywhere. + */ +export function extractGroundingDatums(finalAnswer: string, instruction = ""): ParsedDatum[] { + const datums: ParsedDatum[] = []; + const seen = new Set(); + const normalizedInstruction = normalizeForGrounding(instruction); + const push = (datum: ParsedDatum) => { + const normalized = normalizeForGrounding(datum.text); + const key = `${datum.kind}:${normalized}`; + if (seen.has(key)) return; + seen.add(key); + const fromInstruction = + normalizedInstruction.length > 0 && normalizedInstruction.includes(normalized); + datums.push(fromInstruction ? { ...datum, gates: false, fromInstruction } : datum); + }; + + let remaining = finalAnswer; + for (const match of finalAnswer.matchAll(TIME_TOKEN)) { + const text = match[0]; + push({ + text, + kind: "time", + patterns: [new RegExp(`(?= 3; + const barePattern = new RegExp(`(? { + const a = escapeRegExp(alias); + // "sgd5", "sgd 5.00", "5sgd", "5.00 sgd" + return [ + new RegExp(`${a}\\s?${coreRe}(?:\\.0+)?(?!\\d)`), + new RegExp(`(?= 4 && !YEAR_TOKEN.test(core), + }); + } + + for (const match of finalAnswer.matchAll(ENTITY_TOKEN)) { + const text = match[0].trim(); + if (text.split(/\s+/).length < 2) continue; + push({ + text, + kind: "entity", + patterns: [new RegExp(spaceTolerant(text))], + gates: false, + }); + } + + return datums; +} + +/** Best-effort page URL for a step, carried forward from earlier steps when absent. */ +export function stepUrlHint(step: TrajectoryStep): string | undefined { + const probeUrl = step.probeEvidence?.url; + if (typeof probeUrl === "string" && probeUrl) return probeUrl; + const fromArgs = firstUrl(safeStringify(step.actionArgs)); + if (fromArgs) return fromArgs; + return firstUrl(toolOutputText(step)); +} + +function firstUrl(text: string): string | undefined { + const match = /https?:\/\/[^\s"'\\)\]}>,]+/i.exec(text); + return match?.[0]; +} + +export function isSearchEngineUrl(url: string | undefined): boolean { + if (!url) return false; + let host: string; + try { + host = new URL(url).hostname; + } catch { + const match = /^https?:\/\/([^/?#]+)/i.exec(url); + host = match?.[1] ?? url; + } + return SEARCH_ENGINE_HOST.test(host); +} + +function safeStringify(value: unknown): string { + if (value === undefined || value === null) return ""; + if (typeof value === "string") return value; + try { + return JSON.stringify(value) ?? ""; + } catch { + return String(value); + } +} + +function toolOutputText(step: TrajectoryStep): string { + const parts = [safeStringify(step.toolOutput?.result), step.toolOutput?.error ?? ""]; + for (const modality of step.agentEvidence?.modalities ?? []) { + if (modality.type === "text") parts.push(modality.content); + else if (modality.type === "json") parts.push(safeStringify(modality.content)); + } + return parts.join("\n"); +} + +/** + * Check that every key datum in the final answer appears in at least one step + * output whose page was not a search engine. A number that only shows up in + * Google/Bing snippets (or nowhere) was never verified on the target site. + */ +export function checkAnswerGrounding( + finalAnswer: string, + steps: TrajectoryStep[], + instruction = "", +): GroundingResult | undefined { + const datums = extractGroundingDatums(finalAnswer, instruction); + if (datums.length === 0) return undefined; + + // Steps with no URL hint (a snapshot after navigation) inherit the page of + // the step before them. + const stepTexts: Array<{ text: string; searchEngine: boolean }> = []; + let currentUrl: string | undefined; + for (const step of steps) { + const hint = stepUrlHint(step); + if (hint) currentUrl = hint; + stepTexts.push({ + text: normalizeForGrounding(toolOutputText(step)), + searchEngine: isSearchEngineUrl(currentUrl), + }); + } + + const checked: GroundingDatum[] = datums.map((datum) => { + let groundedAtStep: number | undefined; + let seenInSearch = false; + for (let index = 0; index < stepTexts.length; index += 1) { + const { text, searchEngine } = stepTexts[index]; + if (!datum.patterns.some((pattern) => pattern.test(text))) continue; + if (searchEngine) { + seenInSearch = true; + continue; + } + groundedAtStep = index; + break; + } + return { + text: datum.text, + kind: datum.kind, + ...(groundedAtStep !== undefined && { groundedAtStep }), + onlyInSearchEngine: groundedAtStep === undefined && seenInSearch, + ...(datum.fromInstruction && { fromInstruction: true }), + }; + }); + + const ungrounded = checked.filter((datum) => datum.groundedAtStep === undefined); + const gating = new Set(datums.filter((d) => d.gates).map((d) => `${d.kind}:${d.text}`)); + const isGating = (d: GroundingDatum) => gating.has(`${d.kind}:${d.text}`); + const groundedNumeric = checked.filter( + (d) => isGating(d) && d.groundedAtStep !== undefined, + ).length; + const ungroundedNumeric = ungrounded.filter(isGating).length; + return { + checked, + ungrounded, + groundedNumeric, + ungroundedNumeric, + // Secondary numbers (a comparison price read off a screenshot, a figure + // paraphrased from a snippet) must not sink a row whose headline datum was + // verified on the target site, so the gate needs every numeric datum to be + // ungrounded. Tightening this to "any" flips ~1 in 5 legitimate passes. + gatesOutcome: ungroundedNumeric > 0 && groundedNumeric === 0, + }; +} diff --git a/packages/evals/framework/verifierTrace.ts b/packages/evals/framework/verifierTrace.ts new file mode 100644 index 0000000000..01add9749a --- /dev/null +++ b/packages/evals/framework/verifierTrace.ts @@ -0,0 +1,53 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import type { LogLine } from "stagehand-v3"; + +type TraceLine = LogLine & { parsedAuxiliary?: unknown }; + +/** + * Full verifier trace (EVAL_VERIFIER_TRACE=1). The v3 evaluator itself logs + * only failures, but its LLM client logs every request and response at + * level 2 (category "aisdk"): the batchedRelevance prompt with each evidence + * item, the top-K selection the judge actually reads, and the fusedJudgment + * response. With the switch on, the verifier carrier runs at verbose 2 and + * the grading-phase lines are written to scores/verifier-trace.jsonl instead + * of the row logs (they are megabytes per task). + */ +export const VERIFIER_TRACE_ENV = "EVAL_VERIFIER_TRACE"; +export const VERIFIER_TRACE_FILE = "verifier-trace.jsonl"; +const TRACE_CATEGORIES = new Set(["aisdk", "AISDK error", "verifier", "llm", "flow"]); + +export function verifierTraceEnabled(env: NodeJS.ProcessEnv = process.env): boolean { + const v = env[VERIFIER_TRACE_ENV]?.trim().toLowerCase(); + return v === "1" || v === "true" || v === "yes" || v === "on"; +} + +/** Grading-phase lines worth keeping: the judge's LLM traffic and verifier notes. */ +export function selectVerifierTraceLines( + linesAfter: TraceLine[], + countBefore: number, +): TraceLine[] { + return linesAfter + .slice(countBefore) + .filter((line) => TRACE_CATEGORIES.has(String(line.category ?? ""))); +} + +export async function writeVerifierTrace( + trajectoryDir: string, + lines: TraceLine[], + label?: string, +): Promise { + if (lines.length === 0) return undefined; + const file = path.join( + trajectoryDir, + "scores", + label ? `verifier-trace_${label}.jsonl` : VERIFIER_TRACE_FILE, + ); + try { + await fs.mkdir(path.dirname(file), { recursive: true }); + await fs.writeFile(file, lines.map((line) => JSON.stringify(line)).join("\n") + "\n"); + return file; + } catch { + return undefined; + } +} diff --git a/packages/evals/logger.ts b/packages/evals/logger.ts index 48767010c1..400baf6d90 100644 --- a/packages/evals/logger.ts +++ b/packages/evals/logger.ts @@ -122,11 +122,13 @@ export class EvalLogger { /** * getLogs: - * Retrieves the array of stored log lines. - * Useful for returning logs after a task completes, for analysis or debugging. + * Retrieves the stored log lines at or below `maxLevel` (default 1, so + * level-2 debug lines stay out of persisted task output). Lines without a + * level count as level 1. Pass `{ maxLevel: 2 }` for everything. */ - getLogs(): LogLineEval[] { - return this.logs || []; + getLogs(options: { maxLevel?: number } = {}): LogLineEval[] { + const maxLevel = options.maxLevel ?? 1; + return (this.logs || []).filter((line) => (line.level ?? 1) <= maxLevel); } /** diff --git a/packages/evals/tests/framework/gradeExternalTrajectory.test.ts b/packages/evals/tests/framework/gradeExternalTrajectory.test.ts index 378ac0d1ad..339be2c304 100644 --- a/packages/evals/tests/framework/gradeExternalTrajectory.test.ts +++ b/packages/evals/tests/framework/gradeExternalTrajectory.test.ts @@ -1,7 +1,13 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import type { Rubric, TaskSpec, Trajectory, TrajectoryStep } from "stagehand-v3"; -import { gradeExternalTrajectory } from "../../framework/verifierAdapter.js"; +import { + buildPersistedEvaluationResult, + gradeExternalTrajectory, +} from "../../framework/verifierAdapter.js"; import { EvalLogger } from "../../logger.js"; const mockState = vi.hoisted(() => ({ @@ -28,6 +34,58 @@ vi.mock("stagehand-v3", async (importOriginal) => { }); describe("gradeExternalTrajectory", () => { + it("counts a missing V3 criterion score in the denominator", async () => { + mockState.evaluationResult = { + outcomeSuccess: true, + processScore: 1 / 3, + perCriterion: [ + { criterion: "step one", maxPoints: 1, earnedPoints: 1 }, + { criterion: "step two", maxPoints: 2, earnedPoints: null, evidenceInsufficient: true }, + ], + }; + process.env.EVAL_SUCCESS_MODE = "process"; + const result = await grade({ _success: true }); + expect(result.verifierError).toBeUndefined(); + expect(result.processScoreStrict).toBeCloseTo(1 / 3); + expect(result._success).toBe(false); + }); + + it("persists raw uncertainty and captured evidence without accepting a grade", async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "failed-verifier-evidence-")); + process.env.VERIFIER_PERSIST_TRAJECTORIES = "1"; + mockState.evaluationResult = { + outcomeSuccess: false, + processScore: 0, + findings: [{ category: "verifier_uncertainty", summary: "provider unavailable" }], + }; + try { + const result = await gradeExternalTrajectory({ + buildTrajectory: () => trajectory, + verifier: { v3: {} as never, taskSpec, dataset: "test", trajectoryRoot: root }, + baseResult: { _success: true }, + errorMessage: "unverified", + category: "fixture", + logger: new EvalLogger(false), + }); + expect(result._success).toBe(false); + expect(result.agentReportedSuccess).toBe(true); + expect(result.verifierError).toContain("uncertainty"); + const dir = String(result.trajectoryDir); + expect( + JSON.parse(await fs.readFile(path.join(dir, "trajectory.json"), "utf8")).steps, + ).toHaveLength(3); + expect( + JSON.parse(await fs.readFile(path.join(dir, "scores", "result.json"), "utf8")).findings[0] + .category, + ).toBe("verifier_uncertainty"); + expect( + JSON.parse(await fs.readFile(path.join(dir, "scores", "verifier-error.json"), "utf8")) + .graded, + ).toBe(false); + } finally { + await fs.rm(root, { recursive: true, force: true }); + } + }); const rubric: Rubric = { items: [ { criterion: "step one", description: "does step one", maxPoints: 1 }, @@ -43,7 +101,14 @@ describe("gradeExternalTrajectory", () => { const trajectory = { task: taskSpec, - steps: [{}, {}, {}] as TrajectoryStep[], + steps: Array.from({ length: 3 }, () => ({ + actionName: "stagehand__run", + actionArgs: {}, + reasoning: "", + agentEvidence: { modalities: [{ type: "text", content: "captured page evidence" }] }, + probeEvidence: {}, + toolOutput: { ok: true, result: "captured page evidence" }, + })) as TrajectoryStep[], status: "complete", finalAnswer: "done", usage: {}, @@ -64,6 +129,7 @@ describe("gradeExternalTrajectory", () => { let savedPersist: string | undefined; beforeEach(() => { + mockState.evaluationResult = { outcomeSuccess: true, processScore: 0.92 }; savedSuccessMode = process.env.EVAL_SUCCESS_MODE; savedPersist = process.env.VERIFIER_PERSIST_TRAJECTORIES; delete process.env.EVAL_SUCCESS_MODE; @@ -113,4 +179,175 @@ describe("gradeExternalTrajectory", () => { expect(result._success).toBe(true); expect(result.processScore).toBe(0.95); }); + + it("surfaces the judge verdict, gate list and gate metrics on the result", async () => { + const result = await grade({ metrics: { harness_total_tokens: { count: 1, value: 10 } } }); + + expect(result.judgeOutcomeSuccess).toBe(true); + expect(result.outcomeGates).toEqual([]); + expect(result.processScoreLenient).toBe(0.92); + expect(result.processScoreStrict).toBe(0.92); + expect(result.scoringIncomplete).toBe(true); + const metrics = result.metrics as Record; + expect(metrics.harness_total_tokens).toEqual({ count: 1, value: 10 }); + expect(metrics.outcome_gated).toEqual({ count: 1, value: 0 }); + expect(metrics.scoring_incomplete).toEqual({ count: 1, value: 1 }); + expect(metrics.answer_grounded).toBeUndefined(); + }); + + it("gates a judge pass that never touched the browser and fails _success", async () => { + const result = await gradeExternalTrajectory({ + buildTrajectory: () => + ({ + ...trajectory, + steps: [ + { actionName: "web_fetch", actionArgs: {}, toolOutput: { ok: true, result: "" } }, + ], + }) as unknown as Trajectory, + verifier: { v3: {} as never, taskSpec, dataset: "test" }, + baseResult: { _success: true }, + errorMessage: "agent reported failure", + category: "fx", + logger: new EvalLogger(false), + isFacadeTool: (name) => name.startsWith("stagehand"), + }); + + expect(result.judgeOutcomeSuccess).toBe(true); + expect(result.outcomeSuccess).toBe(false); + expect(result.outcomeGates).toEqual(["no_browser_use"]); + expect(result._success).toBe(false); + // A gate that overrides a judge pass names itself in the row error so the + // reason is visible where the row is read, with the agent's claim attached. + expect(result.error).toMatch( + /^gated: no_browser_use — no browser tool calls \(judge passed; agent said: agent reported failure\)$/, + ); + expect((result.metrics as Record).outcome_gated.value).toBe(1); + }); + + it("persists the gated verdict at the top level of scores/result.json", async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "gated-result-json-")); + process.env.VERIFIER_PERSIST_TRAJECTORIES = "1"; + try { + const result = await gradeExternalTrajectory({ + buildTrajectory: () => + ({ + ...trajectory, + finalAnswer: "", + status: "error", + steps: [ + { + actionName: "stagehand__run", + actionArgs: {}, + reasoning: "", + agentEvidence: { modalities: [] }, + probeEvidence: {}, + toolOutput: { ok: true, result: "" }, + }, + ], + }) as Trajectory, + verifier: { v3: {} as never, taskSpec, dataset: "test", trajectoryRoot: root }, + baseResult: { _success: true }, + errorMessage: "agent reported failure", + category: "eve", + logger: new EvalLogger(false), + }); + expect(result.verifierError).toBeUndefined(); + expect(result.outcomeGates).toEqual(["no_final_answer"]); + + const dir = result.trajectoryDir as string; + const persisted = JSON.parse( + await fs.readFile(path.join(dir, "scores", "result.json"), "utf8"), + ); + expect(persisted).toMatchObject({ + outcomeSuccess: false, + judgeOutcomeSuccess: true, + outcomeGates: ["no_final_answer"], + processScore: 0.92, + processScoreStrict: 0.92, + processScoreLenient: 0.92, + judge: { outcomeSuccess: true, processScore: 0.92 }, + }); + const taskData = JSON.parse(await fs.readFile(path.join(dir, "task_data.json"), "utf8")); + expect(taskData.result.outcomeSuccess).toBe(false); + // The sidecar audit file stays. + const gates = JSON.parse(await fs.readFile(path.join(dir, "scores", "gates.json"), "utf8")); + expect(gates.outcomeSuccess).toBe(false); + } finally { + await fs.rm(root, { recursive: true, force: true }); + } + }); + + it("builds the persisted result from the judge verdict and the gates", () => { + const persisted = buildPersistedEvaluationResult( + { outcomeSuccess: true, processScore: 0.8, perCriterion: [], evidenceInsufficient: ["x"] }, + { + outcomeSuccess: false, + judgeOutcomeSuccess: true, + outcomeGates: ["no_browser_use"], + processScore: 0.5, + processScoreStrict: 0.5, + processScoreLenient: 0.8, + perCriterion: [ + { criterion: "a", maxPoints: 1, earnedPoints: 0, explanation: "", blocked: true }, + ], + blockedCriteria: 1, + scoringIncomplete: false, + }, + ); + expect(persisted.outcomeSuccess).toBe(false); + expect(persisted.judgeOutcomeSuccess).toBe(true); + expect(persisted.processScore).toBe(0.5); + expect(persisted.processScoreLenient).toBe(0.8); + expect(persisted.perCriterion).toEqual([expect.objectContaining({ blocked: true })]); + expect(persisted.evidenceInsufficient).toEqual(["x"]); + expect(persisted.judge).toEqual({ + outcomeSuccess: true, + processScore: 0.8, + perCriterion: [], + evidenceInsufficient: ["x"], + }); + }); + + it("records grounding as advisory by default and gates only when opted in", async () => { + const searchOnly = { + ...trajectory, + finalAnswer: "The seat costs SGD 5.", + steps: [ + { + actionName: "stagehand__run", + actionArgs: { code: "await page.goto('https://www.google.com/search?q=seat')" }, + probeEvidence: { url: "https://www.google.com/search?q=seat" }, + toolOutput: { ok: true, result: "AirAsia seat SGD 5" }, + }, + ], + } as unknown as Trajectory; + const run = (dataset: string) => + gradeExternalTrajectory({ + buildTrajectory: () => searchOnly, + verifier: { v3: {} as never, taskSpec, dataset }, + baseResult: { _success: true }, + errorMessage: "agent reported failure", + category: "eve", + logger: new EvalLogger(false), + }); + + // Default: a correct answer sourced from a search snippet still passes; + // the grounding result is recorded so snippet-sourced passes stay filterable. + const advisory = await run("hardbenchmark"); + expect(advisory.outcomeGates).toEqual([]); + expect(advisory._success).toBe(true); + expect((advisory.metrics as Record).answer_grounded.value).toBe(0); + expect( + (advisory.grounding as { ungrounded: Array<{ text: string }> }).ungrounded[0]?.text, + ).toBe("SGD 5"); + + process.env.EVAL_REQUIRE_GROUNDING = "1"; + try { + const gated = await run("hardbenchmark"); + expect(gated.outcomeGates).toEqual(["ungrounded_answer"]); + expect(gated._success).toBe(false); + } finally { + delete process.env.EVAL_REQUIRE_GROUNDING; + } + }); }); diff --git a/packages/evals/tests/framework/harnessObservations.test.ts b/packages/evals/tests/framework/harnessObservations.test.ts index 6c9b65c79a..39a3fb804a 100644 --- a/packages/evals/tests/framework/harnessObservations.test.ts +++ b/packages/evals/tests/framework/harnessObservations.test.ts @@ -9,6 +9,7 @@ import { } from "../../framework/observationRecorder.js"; import { armsOverLimit, + armsWithPassesWithoutBrowserUse, armsWithUngradedRuns, resolveUnverifiableCriteriaLimit, summarizeArmVerifiability, @@ -238,6 +239,7 @@ describe("verifiability gate", () => { ungradedRuns: 0, unverifiableCriteria: 1, totalCriteria: 7, + passesWithoutBrowserUse: 0, }, { arm: "claude_code × playwright_code × model-a", @@ -245,6 +247,7 @@ describe("verifiability gate", () => { ungradedRuns: 0, unverifiableCriteria: 2, totalCriteria: 5, + passesWithoutBrowserUse: 0, }, ]); }); @@ -254,8 +257,22 @@ describe("verifiability gate", () => { process.env.EVAL_MAX_UNVERIFIABLE_CRITERIA = "1"; expect(resolveUnverifiableCriteriaLimit()).toBe(1); const arms = [ - { arm: "a", gradedRuns: 1, ungradedRuns: 0, unverifiableCriteria: 1, totalCriteria: 4 }, - { arm: "b", gradedRuns: 1, ungradedRuns: 0, unverifiableCriteria: 2, totalCriteria: 4 }, + { + arm: "a", + gradedRuns: 1, + ungradedRuns: 0, + unverifiableCriteria: 1, + totalCriteria: 4, + passesWithoutBrowserUse: 0, + }, + { + arm: "b", + gradedRuns: 1, + ungradedRuns: 0, + unverifiableCriteria: 2, + totalCriteria: 4, + passesWithoutBrowserUse: 0, + }, ]; expect(armsOverLimit(arms, 1).map((a) => a.arm)).toEqual(["b"]); }); @@ -280,6 +297,42 @@ describe("verifiability gate", () => { expect(armsWithUngradedRuns(arms).map((a) => a.arm)).toEqual([arms[0].arm]); }); + it("counts passes that never called the mounted browser surface", () => { + const arms = summarizeArmVerifiability( + [ + row("m", "stagehand_facade", { + criterionCount: 3, + evidenceInsufficient: [], + _success: true, + metrics: { facade_tool_calls: { count: 1, value: 0 } }, + }), + row("m", "stagehand_facade", { + criterionCount: 3, + evidenceInsufficient: [], + _success: true, + metrics: { facade_tool_calls: { count: 1, value: 4 } }, + }), + row("m", "stagehand_facade", { + criterionCount: 3, + evidenceInsufficient: [], + _success: false, + metrics: { facade_tool_calls: { count: 1, value: 0 } }, + }), + // No facade metric at all (older rows): not counted either way. + row("m", "stagehand_facade", { + criterionCount: 3, + evidenceInsufficient: [], + _success: true, + }), + ], + "codex", + ); + expect(arms).toHaveLength(1); + expect(arms[0].gradedRuns).toBe(4); + expect(arms[0].passesWithoutBrowserUse).toBe(1); + expect(armsWithPassesWithoutBrowserUse(arms).map((a) => a.arm)).toEqual([arms[0].arm]); + }); + it("treats malformed limit values as report-only", () => { for (const raw of ["1.5", "10foo", "-2", "", " "]) { process.env.EVAL_MAX_UNVERIFIABLE_CRITERIA = raw; diff --git a/packages/evals/tests/framework/persistTrajectory.test.ts b/packages/evals/tests/framework/persistTrajectory.test.ts index 2384cfb132..6a834c9795 100644 --- a/packages/evals/tests/framework/persistTrajectory.test.ts +++ b/packages/evals/tests/framework/persistTrajectory.test.ts @@ -82,6 +82,32 @@ describe("persistAdapterTrajectory", () => { await fs.rm(tmpRoot, { recursive: true, force: true }); } }); + + it("records terminationReason in trajectory.json and metadata.json", async () => { + const tmpRoot = await fs.mkdtemp(path.join(os.tmpdir(), "persist-adapter-termination-")); + try { + const taskSpec: TaskSpec = { id: "budget-task", instruction: "Test task" }; + const { directory } = await persistAdapterTrajectory({ + trajectory: { + ...makeTrajectory(taskSpec), + status: "error", + terminationReason: "step_budget", + }, + taskSpec, + outputRoot: tmpRoot, + runId: "budget-run", + persist: true, + }); + const trajectory = JSON.parse( + await fs.readFile(path.join(directory, "trajectory.json"), "utf8"), + ); + expect(trajectory).toMatchObject({ status: "error", terminationReason: "step_budget" }); + const metadata = JSON.parse(await fs.readFile(path.join(directory, "metadata.json"), "utf8")); + expect(metadata).toMatchObject({ status: "error", terminationReason: "step_budget" }); + } finally { + await fs.rm(tmpRoot, { recursive: true, force: true }); + } + }); }); function makeTrajectory(task: TaskSpec): Trajectory { diff --git a/packages/evals/tests/framework/verifierGates.test.ts b/packages/evals/tests/framework/verifierGates.test.ts new file mode 100644 index 0000000000..e0651966c0 --- /dev/null +++ b/packages/evals/tests/framework/verifierGates.test.ts @@ -0,0 +1,668 @@ +import { describe, expect, it } from "vitest"; +import type { CriterionScore, EvaluationResult, TrajectoryStep } from "stagehand-v3"; + +import { + applyVerdictGates, + checkAnswerGrounding, + extractGroundingDatums, + isSearchEngineUrl, + resolveRequireGrounding, + strictProcessScore, +} from "../../framework/verifierGates.js"; + +type StepInit = { + action?: string; + /** probeEvidence.url after the step. */ + url?: string; + /** actionArgs.code (a facade `run` call); URLs inside are used as hints. */ + code?: string; + output?: unknown; + ok?: boolean; +}; + +function step({ + action = "stagehand__run", + url, + code, + output = "", + ok = true, +}: StepInit): TrajectoryStep { + return { + actionName: action, + actionArgs: code ? { code } : {}, + reasoning: "", + agentEvidence: { modalities: [] }, + probeEvidence: url ? { url } : {}, + toolOutput: { ok, result: output }, + }; +} + +function criterion( + name: string, + earned: number | null, + max: number, + explanation = "", + extra: Partial = {}, +): CriterionScore { + return { criterion: name, maxPoints: max, earnedPoints: earned, explanation, ...extra }; +} + +const passingJudge: EvaluationResult = { + outcomeSuccess: true, + processScore: 1, + perCriterion: [ + criterion("find it", 2, 2, "Found on the page."), + criterion("report", 1, 1, "Reported."), + ], + evidenceInsufficient: [], +}; + +const isFacadeTool = (name: string) => name.startsWith("stagehand__"); + +describe("applyVerdictGates — outcome gates", () => { + const goodSteps = [step({ url: "https://www.example.com/", output: "Total: $18.95" })]; + + it("leaves a clean judge pass alone", () => { + const gates = applyVerdictGates({ + evaluation: passingJudge, + trajectory: { steps: goodSteps, status: "complete", finalAnswer: "It costs $18.95." }, + isFacadeTool, + requireGrounding: true, + }); + expect(gates.outcomeSuccess).toBe(true); + expect(gates.judgeOutcomeSuccess).toBe(true); + expect(gates.outcomeGates).toEqual([]); + }); + + it("gates a pass with an empty final answer", () => { + const gates = applyVerdictGates({ + evaluation: passingJudge, + trajectory: { steps: goodSteps, status: "complete", finalAnswer: " " }, + requireGrounding: false, + }); + expect(gates.outcomeSuccess).toBe(false); + expect(gates.judgeOutcomeSuccess).toBe(true); + expect(gates.outcomeGates).toEqual(["no_final_answer"]); + }); + + it("preserves a supported completion when execution later ended in error", () => { + const gates = applyVerdictGates({ + evaluation: passingJudge, + trajectory: { steps: goodSteps, status: "error", finalAnswer: "done" }, + requireGrounding: false, + }); + expect(gates.outcomeGates).toEqual([]); + expect(gates.outcomeSuccess).toBe(true); + }); + + it("gates a pass with zero facade tool calls only when a matcher is supplied", () => { + const trajectory = { + steps: [step({ action: "web_fetch", output: "price $18.95" })], + status: "complete" as const, + finalAnswer: "It costs $18.95.", + }; + const withMatcher = applyVerdictGates({ + evaluation: passingJudge, + trajectory, + isFacadeTool, + requireGrounding: false, + }); + expect(withMatcher.outcomeGates).toEqual(["no_browser_use"]); + + const withoutMatcher = applyVerdictGates({ + evaluation: passingJudge, + trajectory, + requireGrounding: false, + }); + expect(withoutMatcher.outcomeGates).toEqual([]); + expect(withoutMatcher.outcomeSuccess).toBe(true); + }); + + it("gates an answer whose only numbers came from search-engine pages", () => { + const trajectory = { + steps: [ + step({ + url: "https://www.google.com/search?q=seat+fee", + output: "AirAsia standard seat SGD 5 per sector", + }), + step({ url: "https://www.airasia.com/flights/", output: "Book flights" }), + ], + status: "complete" as const, + finalAnswer: "A window seat costs SGD 5 per sector.", + }; + const strict = applyVerdictGates({ + evaluation: passingJudge, + trajectory, + requireGrounding: true, + }); + expect(strict.outcomeGates).toEqual(["ungrounded_answer"]); + expect(strict.grounding?.ungrounded.map((d) => d.text)).toEqual(["SGD 5"]); + expect(strict.grounding?.ungrounded[0]?.onlyInSearchEngine).toBe(true); + + const lenient = applyVerdictGates({ + evaluation: passingJudge, + trajectory, + requireGrounding: false, + }); + expect(lenient.outcomeGates).toEqual([]); + expect(lenient.grounding?.gatesOutcome).toBe(true); + }); + + it("never flips a judge fail and reports no gates for it", () => { + const gates = applyVerdictGates({ + evaluation: { ...passingJudge, outcomeSuccess: false }, + trajectory: { steps: [], status: "error", finalAnswer: "" }, + isFacadeTool, + requireGrounding: true, + }); + expect(gates.outcomeSuccess).toBe(false); + expect(gates.judgeOutcomeSuccess).toBe(false); + expect(gates.outcomeGates).toEqual([]); + }); + + it("stacks multiple gates", () => { + const gates = applyVerdictGates({ + evaluation: passingJudge, + trajectory: { steps: goodSteps, status: "error", finalAnswer: "" }, + requireGrounding: true, + }); + expect(gates.outcomeGates).toEqual(["no_final_answer"]); + }); +}); + +describe("strictProcessScore", () => { + it("reports blocker wording without inferring missing evidence", () => { + const { perCriterion, strict, blockedCriteria } = strictProcessScore({ + perCriterion: [ + criterion("search", 4, 4, "Configured the search with all constraints."), + criterion( + "seat cost", + 5, + 5, + "Due to an uncontrollable platform blocker the seat map was unreachable. Full credit awarded.", + ), + criterion("report", 3, 3, "Reported correctly."), + ], + evidenceInsufficient: [], + }); + expect(strict).toBe(1); + expect(blockedCriteria).toBe(0); + expect(perCriterion?.[1]?.blockerMentioned).toBe(true); + expect(perCriterion?.[0]?.blocked).toBeUndefined(); + }); + + it("does not mark a partially-credited criterion as blocked even if it mentions a blocker", () => { + const { perCriterion, strict } = strictProcessScore({ + perCriterion: [ + criterion("seat cost", 3, 5, "Blocked midway; partial credit for the attempt."), + ], + }); + expect(perCriterion?.[0]?.blocked).toBeUndefined(); + expect(strict).toBeCloseTo(0.6); + }); + + it("zeroes evidenceInsufficient criteria via the flag or the top-level list", () => { + const { strict, blockedCriteria } = strictProcessScore({ + perCriterion: [ + criterion("a", 2, 2, "ok", { evidenceInsufficient: true }), + criterion("b", 2, 2, "ok"), + criterion("c", 2, 2, "ok"), + ], + evidenceInsufficient: ["c"], + }); + expect(blockedCriteria).toBe(2); + expect(strict).toBeCloseTo(2 / 6); + }); + + it("excludes not-applicable criteria and zeros explicitly unsupported work", () => { + const blocked = "Access Denied wall at step 0. Full credit due to uncontrollable blocker."; + const { strict, blockedCriteria } = strictProcessScore({ + perCriterion: [ + criterion("locate", 2, 2, blocked, { evidenceInsufficient: true }), + criterion("add to cart", 3, 3, blocked, { evidenceInsufficient: true }), + criterion("conditional", null, 2, "Not applicable."), + ], + }); + expect(blockedCriteria).toBe(2); + expect(strict).toBe(0); + }); + + it("returns undefined without perCriterion", () => { + expect(strictProcessScore({})).toEqual({ + perCriterion: undefined, + strict: undefined, + blockedCriteria: 0, + }); + }); +}); + +describe("grounding — datum extraction", () => { + it("extracts currency, percent, time, decimal and long-integer datums as gating", () => { + const datums = extractGroundingDatums( + "Costs SGD 5 or $18.95 (12% off), zip 11222, 2.4 miles, lap 1:27:02.624, 3 stops, in 2023.", + ); + const byText = Object.fromEntries(datums.map((d) => [d.text, d])); + expect(byText["SGD 5"]).toMatchObject({ kind: "currency", gates: true }); + expect(byText["$18.95"]).toMatchObject({ kind: "currency", gates: true }); + expect(byText["12%"]).toMatchObject({ kind: "percent", gates: true }); + expect(byText["1:27:02.624"]).toMatchObject({ kind: "time", gates: true }); + expect(byText["2.4"]).toMatchObject({ kind: "decimal", gates: true }); + expect(byText["11222"]).toMatchObject({ kind: "integer", gates: true }); + expect(byText["3"]).toMatchObject({ kind: "integer", gates: false }); + expect(byText["2023"]).toMatchObject({ kind: "integer", gates: false }); + }); + + it("treats capitalised multi-word entities as advisory", () => { + const datums = extractGroundingDatums("Max Verstappen won the Abu Dhabi Grand Prix."); + const entities = datums.filter((d) => d.kind === "entity").map((d) => d.text); + expect(entities).toContain("Max Verstappen"); + expect(entities).toContain("Abu Dhabi Grand Prix"); + expect(datums.every((d) => d.kind !== "entity" || d.gates === false)).toBe(true); + }); + + it("never gates on a datum echoed from the task instruction", () => { + const datums = extractGroundingDatums( + "Under $15: CVS gummies at $8.99.", + "Find CVS multivitamins under $15", + ); + const byText = Object.fromEntries(datums.map((d) => [d.text, d])); + expect(byText["$15"]).toMatchObject({ gates: false, fromInstruction: true }); + expect(byText["$8.99"]).toMatchObject({ gates: true }); + }); +}); + +describe("grounding — matching", () => { + it("matches currency with alias, spacing and trailing zeros", () => { + const steps = [step({ url: "https://shop.example.com", output: "Seat fee: S$ 5.00 each" })]; + const result = checkAnswerGrounding("SGD 5 per sector", steps); + expect(result?.ungrounded).toEqual([]); + expect(result?.checked[0]?.groundedAtStep).toBe(0); + }); + + it("matches numbers regardless of thousands separators and case", () => { + const steps = [step({ url: "https://data.example.com", output: "POPULATION: 8336817 (est.)" })]; + expect(checkAnswerGrounding("about 8,336,817 people", steps)?.gatesOutcome).toBe(false); + }); + + it("does not let a currency amount match inside a longer number", () => { + const steps = [step({ url: "https://shop.example.com", output: "Total $118.95" })]; + expect(checkAnswerGrounding("costs $18.95", steps)?.gatesOutcome).toBe(true); + }); + + it("carries the page URL forward to steps without a hint", () => { + const steps = [ + step({ code: "await page.goto('https://www.google.com/search?q=x')", output: "results" }), + step({ action: "stagehand__snapshot", output: "AI Overview: the fee is $42.50" }), + ]; + const result = checkAnswerGrounding("fee is $42.50", steps); + expect(result?.gatesOutcome).toBe(true); + expect(result?.ungrounded[0]?.onlyInSearchEngine).toBe(true); + }); + + it("prefers the probe URL over URLs mentioned in the code or output", () => { + const steps = [ + step({ + url: "https://www.target-site.com/results", + code: "await page.goto('https://www.google.com/search?q=x'); await page.click('a')", + output: "target-site result: $42.50", + }), + ]; + expect(checkAnswerGrounding("fee is $42.50", steps)?.gatesOutcome).toBe(false); + }); + + it("does not gate when the headline datum is grounded but a secondary one is not", () => { + const steps = [step({ url: "https://www.ups.com/rates", output: "Medium box from $18.95" })]; + const result = checkAnswerGrounding("UPS medium: $18.95. FedEx is around $24.35.", steps); + expect(result?.groundedNumeric).toBe(1); + expect(result?.ungroundedNumeric).toBe(1); + expect(result?.gatesOutcome).toBe(false); + }); + + it("returns undefined when the answer has nothing to check", () => { + expect(checkAnswerGrounding("done", [])).toBeUndefined(); + }); + + it("does not gate an answer with only entity datums", () => { + const result = checkAnswerGrounding("Canyonlands National Park", []); + expect(result?.gatesOutcome).toBe(false); + expect(result?.ungrounded).toHaveLength(1); + }); + + it("classifies search-engine hosts by domain, including subdomains", () => { + expect(isSearchEngineUrl("https://www.google.com/search?q=a")).toBe(true); + expect(isSearchEngineUrl("https://html.duckduckgo.com/html/?q=a")).toBe(true); + expect(isSearchEngineUrl("https://www.scribd.com/doc/1")).toBe(true); + expect(isSearchEngineUrl("https://old.reddit.com/r/x")).toBe(true); + expect(isSearchEngineUrl("https://www.googleapis-mirror.example.com/")).toBe(false); + expect(isSearchEngineUrl("https://www.airasia.com/")).toBe(false); + expect(isSearchEngineUrl(undefined)).toBe(false); + }); +}); + +describe("scoringIncomplete", () => { + it("flags a rubric with more items than judged criteria without touching the outcome", () => { + const gates = applyVerdictGates({ + evaluation: passingJudge, + trajectory: { + steps: [step({ url: "https://www.example.com", output: "$18.95" })], + status: "complete", + finalAnswer: "$18.95", + }, + requireGrounding: true, + rubricItemCount: 3, + }); + expect(gates.scoringIncomplete).toBe(true); + expect(gates.outcomeSuccess).toBe(true); + }); + + it("is false when counts match or no rubric count is known", () => { + const base = { + evaluation: passingJudge, + trajectory: { steps: [] as TrajectoryStep[], status: "complete" as const, finalAnswer: "ok" }, + requireGrounding: false, + }; + expect(applyVerdictGates({ ...base, rubricItemCount: 2 }).scoringIncomplete).toBe(false); + expect(applyVerdictGates(base).scoringIncomplete).toBe(false); + }); +}); + +describe("resolveRequireGrounding", () => { + it("honours the env override in both directions", () => { + expect(resolveRequireGrounding("hardbenchmark", true, { EVAL_REQUIRE_GROUNDING: "0" })).toBe( + false, + ); + expect(resolveRequireGrounding("custom", false, { EVAL_REQUIRE_GROUNDING: "1" })).toBe(true); + }); + + it("is advisory by default and gates only with EVAL_REQUIRE_GROUNDING=1", () => { + expect(resolveRequireGrounding("hardbenchmark", false, {})).toBe(false); + expect(resolveRequireGrounding("custom", true, {})).toBe(false); + expect(resolveRequireGrounding("hardbenchmark", true, { EVAL_REQUIRE_GROUNDING: "1" })).toBe( + true, + ); + expect(resolveRequireGrounding("hardbenchmark", true, { EVAL_REQUIRE_GROUNDING: "0" })).toBe( + false, + ); + }); +}); + +/** + * Minimal excerpts of the audited HardBench rows (2026-08-31, gpt-5.6-luna). + * Each keeps only the fields the gates read; toolOutput text is trimmed to the + * fragment that carried the datum. + */ +describe("evidence rows from the 2026-08-31 audit", () => { + it("deepagents 7e6993f2 (imgur meme): pass with status=error and no answer is gated", () => { + const evaluation: EvaluationResult = { + outcomeSuccess: true, + processScore: 1, + perCriterion: [ + criterion("Include a frog as the background image", 3, 3, "Frog head as background."), + criterion('Add the exact text "Enjoy your life"', 3, 3, "Text added, centered."), + criterion('Ensure "Enjoy your life" is the only text on the meme', 2, 2, "No other text."), + criterion("Present the created meme", 2, 2, "Presented on the editor canvas."), + ], + evidenceInsufficient: [], + }; + const steps = Array.from({ length: 51 }, () => + step({ + action: "stagehand.snapshot", + url: "https://imgur.com/", + output: "[3-5] RootWebArea: Imgur: The magic of the Internet", + }), + ); + const gates = applyVerdictGates({ + evaluation, + trajectory: { steps, status: "error", finalAnswer: "" }, + isFacadeTool: (name) => name.startsWith("stagehand."), + requireGrounding: true, + rubricItemCount: 4, + }); + expect(gates.judgeOutcomeSuccess).toBe(true); + expect(gates.outcomeSuccess).toBe(false); + expect(gates.outcomeGates).toEqual(["no_final_answer"]); + expect(gates.processScoreStrict).toBe(1); + expect(gates.grounding).toBeUndefined(); + }); + + it("eve airasia_88: SGD 5 lifted from Google snippets is gated; blocker credit zeroed", () => { + const evaluation: EvaluationResult = { + outcomeSuccess: true, + processScore: 11 / 12, + perCriterion: [ + criterion( + "Search for AirAsia flights with the correct constraints", + 4, + 4, + "Configured the search on AirAsia with all required constraints.", + ), + criterion( + "Determine window-seat selection cost for the matching itinerary", + 4, + 5, + "Due to an uncontrollable blocker on the results page the agent could not reach the seat map. However, it resolved the fee via search.", + ), + criterion( + "Report unavailability if no matching direct AirAsia flights", + 3, + 3, + "Not applicable because direct flights exist; full points awarded per instructions.", + ), + ], + evidenceInsufficient: [], + }; + const steps = [ + step({ + code: "await page.goto('https://www.google.com/search?q=AirAsia+Singapore+to+Langkawi+booking')", + url: "https://www.google.com/search?q=AirAsia+Singapore+to+Langkawi+booking", + output: + "airasia.com › flights AirAsia standard seat from SGD 5 ... https://www.airasia.com", + }), + step({ + code: "await page.goto('https://www.airasia.com/flights/')", + url: "https://www.airasia.com/flights/", + output: "One-way Round-trip Singapore Changi Airport Langkawi 24/11/2026 27/11/2026", + }), + step({ + url: "https://www.airasia.com/v2/flights/search/?origin=SIN&destination=LGK", + output: "Loading flights ... (skeleton)", + }), + step({ + code: "await page.goto('https://www.google.com/search?q=AirAsia+seat+selection+window+seat+fee+Singapore')", + url: "https://www.google.com/search?q=AirAsia+seat+selection+window+seat+fee+Singapore", + output: "Standard seat (window/aisle) SGD 5 – SGD 10 per sector · support.airasia.com", + }), + step({ + url: "https://www.google.com/search?q=%22International+Route+-+Seat+(AK)%22", + output: "scribd.com › document AirAsia fee schedule: Standard Seat SGD 5 ...", + }), + ]; + const gates = applyVerdictGates({ + evaluation, + trajectory: { + task: { + instruction: + "How much does it cost to select a window seat on a direct AirAsia flight from Singapore to Langkawi from November 24 to November 27? If there are no available flights for those dates, please indicate that in your answer", + }, + steps, + status: "complete", + finalAnswer: + "A standard window seat costs SGD 5 per passenger per sector. For the round trip from Singapore to Langkawi (Nov 24–27, 2026), selecting a window seat for both flights costs SGD 10 total. Direct flights are available.", + }, + isFacadeTool, + requireGrounding: true, + rubricItemCount: 3, + }); + expect(gates.judgeOutcomeSuccess).toBe(true); + expect(gates.outcomeGates).toEqual(["ungrounded_answer"]); + expect(gates.outcomeSuccess).toBe(false); + const ungrounded = gates.grounding?.ungrounded.filter((d) => d.kind === "currency"); + expect(ungrounded?.map((d) => d.text)).toEqual(["SGD 5", "SGD 10"]); + expect(ungrounded?.every((d) => d.onlyInSearchEngine)).toBe(true); + // "2026" and "24"/"27" are grounded on airasia.com but must not rescue + // the row: years and short integers never count as grounded numerics. + expect(gates.grounding?.groundedNumeric).toBe(0); + // 4/5 on the blocker criterion is partial credit, not the full-credit + // blocker rule, so the strict score matches the judge here. + expect(gates.processScoreStrict).toBeCloseTo(11 / 12); + expect(gates.processScoreLenient).toBeCloseTo(11 / 12); + }); + + it("eve afcebfed (CVS gluten-free): grounded on cvs.com, not gated", () => { + const evaluation: EvaluationResult = { + outcomeSuccess: true, + processScore: 1, + perCriterion: [ + criterion("Identify multivitamins", 2, 2, "Searched 'gluten free multivitamin'."), + criterion("Apply 'gluten-free' filter or identify attribute", 2, 2, "Search term applied."), + criterion("Apply 'CVS Health Brand' filter or identify brand", 2, 2, "Checked CVS brand."), + criterion("Apply price filter 'under $15'", 2, 2, "Checked $5-$10 and $10-$15."), + criterion("Sort or identify by 'most reviewed'", 3, 3, "Sorted by Most Reviewed."), + criterion("Report the identified product", 3, 3, "Reported the top product."), + ], + evidenceInsufficient: [], + }; + const steps = [ + step({ + code: "await page.goto('https://www.cvs.com/')", + url: "https://www.cvs.com/", + output: "CVS", + }), + step({ + url: "https://www.cvs.com/search?searchTerm=gluten%20free%20multivitamin", + output: + "CVS Women's Daily Multivitamin Gummies, 150 CT $8.99 (280) · CVS Men 50+ Advanced Multivitamin Tablets, 65 CT $9.39 (246) · CVS Women's Multivitamin Tablets, 120 CT $13.49 (243)", + }), + ]; + const gates = applyVerdictGates({ + evaluation, + trajectory: { + task: { + instruction: + "Find the most reviewed gluten-free CVS Health Brand multivitamin under $15 on cvs.com", + }, + steps, + status: "complete", + finalAnswer: + "Most-reviewed matches: 1) CVS Women’s Daily Multivitamin Gummies, 150 CT — $8.99, 280 reviews; 2) CVS Men 50+ Advanced Multivitamin Tablets, 65 CT — $9.39, 246 reviews; 3) CVS Women’s Multivitamin Tablets, 120 CT — $13.49, 243 reviews.", + }, + isFacadeTool, + requireGrounding: true, + rubricItemCount: 6, + }); + // Constraint leaks (gluten-free never verified) are the judge's job; the + // deterministic gates have nothing to say about this row. + expect(gates.outcomeSuccess).toBe(true); + expect(gates.outcomeGates).toEqual([]); + expect(gates.grounding?.groundedNumeric).toBeGreaterThan(0); + expect(gates.grounding?.ungroundedNumeric).toBe(0); + expect(gates.processScoreStrict).toBe(1); + expect(gates.scoringIncomplete).toBe(false); + }); + + it("a0a18ca6 (Macy's bot wall): processScore 1.0 becomes 0 strict", () => { + const blocked = (what: string) => + `${what} was impossible due to the uncontrollable platform blocker; full process credit is awarded.`; + const evaluation: EvaluationResult = { + outcomeSuccess: false, + processScore: 1, + perCriterion: [ + criterion( + "Locate a men's T-shirt", + 2, + 2, + "Blocked immediately by Macy's anti-bot ('Access Denied').", + ), + criterion("Select 'large' size for the T-shirt", 2, 2, blocked("Selecting the size")), + criterion( + "Apply 'stripe pattern' filter for the T-shirt", + 2, + 2, + blocked("Applying the filter"), + ), + criterion( + "Apply 'short sleeve' filter for the T-shirt", + 2, + 2, + blocked("Applying the filter"), + ), + criterion("Select a T-shirt from the 'Best Sellers' group", 2, 2, blocked("Selecting")), + criterion("Add the selected T-shirt to the cart", 3, 3, blocked("Adding to cart")), + criterion( + "Respect Critical Point boundaries", + 2, + 2, + "Did not cross any transactional boundary as it was blocked from entering the website.", + ), + ], + evidenceInsufficient: [], + }; + const gates = applyVerdictGates({ + evaluation, + trajectory: { + steps: [step({ url: "https://www.macys.com/", output: "Access Denied" })], + status: "complete", + finalAnswer: "Unable to complete the task because Macy’s blocked access.", + }, + isFacadeTool, + requireGrounding: true, + }); + expect(gates.processScoreLenient).toBe(1); + expect(gates.processScoreStrict).toBe(1); + expect(gates.blockedCriteria).toBe(0); + expect(gates.perCriterion?.every((c) => c.blockerMentioned)).toBe(true); + expect(gates.outcomeSuccess).toBe(false); + }); + + it("mastra 864244b6 (F1): race time seen on espn.com after search detours is grounded", () => { + const steps = [ + step({ + action: "stagehand_run", + code: "await page.goto('https://www.bing.com/search?q=2023+Abu+Dhabi+Grand+Prix+race+time')", + url: "https://www.bing.com/search?q=2023+Abu+Dhabi+Grand+Prix+race+time", + output: '{"title": "2023 Abu Dhabi Grand Prix race time 1:27:02.624 - Search"}', + }), + step({ + action: "stagehand_run", + code: "await page.goto('https://www.espn.com/f1/results/_/id/600026790')", + url: "https://www.espn.com/f1/results/_/id/600026790", + output: + "Etihad Airways Abu Dhabi GP November 24 - November 26, 2023 Yas Marina Circuit RACE WINNER Max Verstappen 1:27:02.624 2 Pits", + }), + ]; + const gates = applyVerdictGates({ + evaluation: passingJudge, + trajectory: { + steps, + status: "complete", + finalAnswer: "Max Verstappen won first place with a race time of 1:27:02.624.", + }, + isFacadeTool: (name) => name === "stagehand_run", + requireGrounding: true, + }); + expect(gates.outcomeSuccess).toBe(true); + expect(gates.outcomeGates).toEqual([]); + expect(gates.grounding?.checked.find((d) => d.kind === "time")?.groundedAtStep).toBe(1); + }); + + it("the same answer with the time only in a Google snippet is gated", () => { + const steps = [ + step({ + url: "https://www.google.com/search?q=ESPN+2023+Abu+Dhabi+Grand+Prix+results", + output: + "ESPN https://www.espn.com › race 2023 Etihad Airways Abu Dhabi Grand Prix F1, won by Max Verstappen. Winner VER 1:27:02.624", + }), + step({ url: "https://www.espn.com/f1/", output: "F1 home — schedule, standings" }), + ]; + const gates = applyVerdictGates({ + evaluation: passingJudge, + trajectory: { + steps, + status: "complete", + finalAnswer: "Max Verstappen won the 2023 Abu Dhabi Grand Prix in 1:27:02.624.", + }, + requireGrounding: true, + }); + expect(gates.outcomeGates).toEqual(["ungrounded_answer"]); + }); +}); diff --git a/packages/evals/tests/logger.test.ts b/packages/evals/tests/logger.test.ts index a33d01db0a..9bf0c9623a 100644 --- a/packages/evals/tests/logger.test.ts +++ b/packages/evals/tests/logger.test.ts @@ -35,4 +35,16 @@ describe("EvalLogger", () => { expect(logSpy).toHaveBeenCalledTimes(1); expect(logger.getLogs()).toHaveLength(1); }); + + it("keeps level-2 debug lines out of getLogs unless asked for", () => { + const logger = new EvalLogger(false); + logger.log({ category: "session", message: "session", level: 0 }); + logger.log({ category: "trace", message: "step", level: 1 }); + logger.log({ category: "trace", message: "unlevelled" }); + logger.log({ category: "mastra", message: "tool-call-delta event", level: 2 }); + + expect(logger.getLogs().map((line) => line.message)).toEqual(["session", "step", "unlevelled"]); + expect(logger.getLogs({ maxLevel: 2 })).toHaveLength(4); + expect(logger.getLogs({ maxLevel: 0 }).map((line) => line.message)).toEqual(["session"]); + }); }); diff --git a/packages/evals/tests/tui/verify.test.ts b/packages/evals/tests/tui/verify.test.ts new file mode 100644 index 0000000000..b037fec6df --- /dev/null +++ b/packages/evals/tests/tui/verify.test.ts @@ -0,0 +1,36 @@ +import { describe, expect, it } from "vitest"; +import { formatVerdictLine } from "../../tui/commands/verify.js"; + +// eslint-disable-next-line no-control-regex +const plain = (value: string) => value.replace(/\[[0-9;]*m/g, ""); + +describe("evals verify verdict line", () => { + it("shows the gated outcome first and names the gates when they flipped the judge", () => { + const line = plain( + formatVerdictLine({ + outcomeSuccess: false, + judgeOutcomeSuccess: true, + outcomeGates: ["no_final_answer", "no_browser_use"], + processScore: 0.5, + processScoreLenient: 0.75, + }), + ); + expect(line).toContain("outcomeSuccess=false"); + expect(line).toContain("judge=true gated=no_final_answer,no_browser_use"); + expect(line).toContain("processScore=0.500 (lenient=0.750)"); + }); + + it("still reports the judge verdict when nothing was gated", () => { + const line = plain( + formatVerdictLine({ + outcomeSuccess: true, + judgeOutcomeSuccess: true, + outcomeGates: [], + processScore: undefined, + processScoreLenient: undefined, + }), + ); + expect(line).toContain("outcomeSuccess=true judge=true"); + expect(line).toContain("processScore=n/a (lenient=n/a)"); + }); +}); diff --git a/packages/evals/tui/commands/verify.ts b/packages/evals/tui/commands/verify.ts index 763ea745b4..94b194cb65 100644 --- a/packages/evals/tui/commands/verify.ts +++ b/packages/evals/tui/commands/verify.ts @@ -5,6 +5,11 @@ * and returns an EvaluationResult. This command reads the on-disk layout written by * `TrajectoryRecorder.persist()` and feeds it through V3Evaluator.verify(). * + * The judge's verdict is then passed through the same deterministic gates the + * live run applies (see verifierGates.ts), so the offline result matches what + * the row would have scored. The facade gate (`no_browser_use`) needs the live + * tool matcher and is not applied offline. + * * Output: writes a new result file under `scores/result_