diff --git a/cloudflare-workers/api-edge/migrations/0004_evals.sql b/cloudflare-workers/api-edge/migrations/0004_evals.sql new file mode 100644 index 00000000..8676d4d2 --- /dev/null +++ b/cloudflare-workers/api-edge/migrations/0004_evals.sql @@ -0,0 +1,61 @@ +-- Native agent evals (input/output). A dataset is a set of {input, expect} examples for one +-- deployed agent; a run executes every example as an isolated session against that agent and +-- scores the output. Runtime-agnostic: the runner drives the /v3 session API, so flue, langgraph, +-- claude, codex and pi are all just "the agent under test". Owned by api-edge + D1 (a consumer of +-- sessions-api, not part of it). + +-- A reusable set of {input, expect} examples for a single agent. +CREATE TABLE IF NOT EXISTS eval_datasets ( + id TEXT PRIMARY KEY, -- evd_ + org_id TEXT NOT NULL, + agent_id TEXT NOT NULL, -- the sessions-api agent id (system under test) + name TEXT NOT NULL, + examples TEXT NOT NULL DEFAULT '[]', -- JSON: [{ id, input, expect?: {contains?,equals?,iregex?,tools?,outcome?,max_cost_usd?} }] + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL +); +CREATE INDEX IF NOT EXISTS idx_eval_datasets_org_agent ON eval_datasets (org_id, agent_id, updated_at DESC); + +-- One execution of a dataset against the agent (pinned to the live revision at run time). +CREATE TABLE IF NOT EXISTS eval_runs ( + id TEXT PRIMARY KEY, -- evr_ + org_id TEXT NOT NULL, + dataset_id TEXT NOT NULL, + agent_id TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'pending', -- pending | running | done | failed + total INTEGER NOT NULL DEFAULT 0, + completed INTEGER NOT NULL DEFAULT 0, -- results in a terminal state + passed INTEGER NOT NULL DEFAULT 0, -- results with every check passing + score REAL, -- passed / completed (0..1), null until any complete + error TEXT, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + finished_at INTEGER +); +CREATE INDEX IF NOT EXISTS idx_eval_runs_dataset ON eval_runs (dataset_id, created_at DESC); +CREATE INDEX IF NOT EXISTS idx_eval_runs_status ON eval_runs (status, updated_at); + +-- Per-example result within a run. Drives a small state machine (pending -> running -> done/failed) +-- so the cron + a request-time kick can resume it without a Durable Object. +CREATE TABLE IF NOT EXISTS eval_results ( + id TEXT PRIMARY KEY, -- evres_ + run_id TEXT NOT NULL, + org_id TEXT NOT NULL, + example_id TEXT NOT NULL, + input TEXT NOT NULL, -- denormalized example input + expect TEXT NOT NULL DEFAULT '{}', -- denormalized example expectations (JSON) + state TEXT NOT NULL DEFAULT 'pending', -- pending | running | done | failed + session_id TEXT, -- the /v3 session created for this example + output TEXT, -- final agent message text + outcome TEXT, -- terminal turn state (ok | error | ...) + cost_usd REAL, + tokens INTEGER, + scores TEXT NOT NULL DEFAULT '[]', -- JSON: [{ name, pass, detail }] + passed INTEGER, -- 0/1: every check passed + error TEXT, + attempts INTEGER NOT NULL DEFAULT 0, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL +); +CREATE INDEX IF NOT EXISTS idx_eval_results_run ON eval_results (run_id, created_at); +CREATE INDEX IF NOT EXISTS idx_eval_results_state ON eval_results (state, updated_at); diff --git a/cloudflare-workers/api-edge/src/dashboard.ts b/cloudflare-workers/api-edge/src/dashboard.ts index 554d3be1..f552490e 100644 --- a/cloudflare-workers/api-edge/src/dashboard.ts +++ b/cloudflare-workers/api-edge/src/dashboard.ts @@ -31,6 +31,7 @@ import { acknowledgeAgentSecurityNotification, listAgentSecurityNotifications, } from "./agent_security_notifications"; +import { handleEvals } from "./evals"; import { createAPIKey } from "./api_keys"; export interface DashboardEnv { @@ -157,7 +158,7 @@ async function mintCellCapToken(secret: string, orgID: string, cellID: string, p // OC_ORG_TOKEN_SECRET (shared with sessions-api). /v3 trusts it and sets owner = // the asserted org — same "act for org X" shape as the cell cap-token, so no // osb_ key reaches the browser and /v3 never custodies a customer key. -async function mintOrgToken(secret: string, orgID: string, userID: string | null): Promise { +export async function mintOrgToken(secret: string, orgID: string, userID: string | null): Promise { const now = Math.floor(Date.now() / 1000); const header = { alg: "HS256", typ: "JWT" }; const payload: Record = { @@ -889,7 +890,7 @@ async function proxyToBrowserAPI( export async function handleDashboard( req: Request, env: DashboardEnv, - _ctx: ExecutionContext, + ctx: ExecutionContext, path: string, ): Promise { const caller = await authDashboard(req, env); @@ -907,6 +908,11 @@ export async function handleDashboard( return proxyToV3(req, env, caller, sub); } + // ── Evals (edge-owned, D1-backed; the runner drives /v3 to run the agent) ── + if (sub === "/evals" || sub.startsWith("/evals/")) { + return handleEvals(req, env, caller, ctx, sub, method); + } + // ── Browser Sessions — proxy to the dedicated browser Worker ─────────── if (sub === "/browsers" && method === "GET") return proxyToBrowserAPI(req, env, caller, "/v1/browsers"); if (sub === "/browser-usage" && method === "GET") return proxyToBrowserAPI(req, env, caller, "/v1/browser-usage"); diff --git a/cloudflare-workers/api-edge/src/evals.test.ts b/cloudflare-workers/api-edge/src/evals.test.ts new file mode 100644 index 00000000..f96ef29e --- /dev/null +++ b/cloudflare-workers/api-edge/src/evals.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it } from "vitest"; +import { scoreOutput } from "./evals"; + +const pass = (scores: { name: string; pass: boolean }[], name: string) => + scores.find((s) => s.name === name)?.pass; + +describe("evals scoreOutput", () => { + it("always scores completion from the turn outcome", () => { + expect(pass(scoreOutput({}, { output: "hi", outcome: "ok", tools: [] }), "completed")).toBe(true); + expect(pass(scoreOutput({}, { output: "", outcome: "error", tools: [] }), "completed")).toBe(false); + }); + + it("contains: case-insensitive, all substrings required", () => { + const obs = { output: "Paris is the capital, and 12*13 = 156.", outcome: "ok", tools: [] }; + expect(pass(scoreOutput({ contains: ["paris", "156"] }, obs), "contains")).toBe(true); + expect(pass(scoreOutput({ contains: ["paris", "999"] }, obs), "contains")).toBe(false); + }); + + it("equals: normalized (trim + case)", () => { + expect(pass(scoreOutput({ equals: "yes" }, { output: " YES ", outcome: "ok", tools: [] }), "equals")).toBe(true); + expect(pass(scoreOutput({ equals: "yes" }, { output: "no", outcome: "ok", tools: [] }), "equals")).toBe(false); + }); + + it("regex: case-insensitive; invalid regex fails safe", () => { + expect(pass(scoreOutput({ iregex: "^\\d+$" }, { output: "42", outcome: "ok", tools: [] }), "regex")).toBe(true); + const bad = scoreOutput({ iregex: "(" }, { output: "x", outcome: "ok", tools: [] }); + expect(pass(bad, "regex")).toBe(false); + expect(bad.find((s) => s.name === "regex")?.detail).toBe("invalid regex"); + }); + + it("tools: every named tool must be called", () => { + const obs = { output: "done", outcome: "ok", tools: ["bash", "read"] }; + expect(pass(scoreOutput({ tools: ["bash"] }, obs), "tools")).toBe(true); + expect(pass(scoreOutput({ tools: ["bash", "write"] }, obs), "tools")).toBe(false); + }); + + it("cost: at or under budget", () => { + expect(pass(scoreOutput({ max_cost_usd: 0.05 }, { output: "x", outcome: "ok", cost_usd: 0.02, tools: [] }), "cost")).toBe(true); + expect(pass(scoreOutput({ max_cost_usd: 0.05 }, { output: "x", outcome: "ok", cost_usd: 0.10, tools: [] }), "cost")).toBe(false); + }); + + it("only emits scores for declared expectations (plus completion)", () => { + const names = scoreOutput({ contains: ["a"] }, { output: "a", outcome: "ok", tools: [] }).map((s) => s.name); + expect(names).toEqual(["completed", "contains"]); + }); +}); diff --git a/cloudflare-workers/api-edge/src/evals.ts b/cloudflare-workers/api-edge/src/evals.ts new file mode 100644 index 00000000..586c8a82 --- /dev/null +++ b/cloudflare-workers/api-edge/src/evals.ts @@ -0,0 +1,426 @@ +// evals.ts — native agent evals (input/output), edge-owned. +// +// A dataset is a set of { input, expect } examples for one deployed agent. A run executes every +// example as an isolated /v3 session against that agent and scores the output. The runner is +// runtime-agnostic — it drives the public session API (POST /v3/sessions → poll GET /result), so +// flue, langgraph, claude, codex and pi are all just "the agent under test". State lives in D1; +// the run advances via a small state machine (pending → running → done/failed) kicked by +// ctx.waitUntil after create and reconciled by the 5-min cron — no Durable Object needed. + +import type { DashboardEnv } from "./dashboard"; +import { mintOrgToken } from "./dashboard"; + +// ── shapes ───────────────────────────────────────────────────────────────── +interface Expect { + contains?: string[]; // every substring present (case-insensitive) + equals?: string; // normalized exact match + iregex?: string; // case-insensitive regex match + outcome?: string; // terminal turn state must equal this (e.g. "ok") + tools?: string[]; // every named tool was called (needs the event trace) + max_cost_usd?: number; // turn cost at or under this +} +interface Example { id: string; input: string; expect?: Expect } +interface Score { name: string; pass: boolean; detail?: string } +interface EvalCaller { orgID: string; userID: string } + +interface DatasetRow { + id: string; org_id: string; agent_id: string; name: string; + examples: string; created_at: number; updated_at: number; +} +interface RunRow { + id: string; org_id: string; dataset_id: string; agent_id: string; status: string; + total: number; completed: number; passed: number; score: number | null; + error: string | null; created_at: number; updated_at: number; finished_at: number | null; +} +interface ResultRow { + id: string; run_id: string; org_id: string; example_id: string; input: string; expect: string; + state: string; session_id: string | null; output: string | null; outcome: string | null; + cost_usd: number | null; tokens: number | null; scores: string; passed: number | null; + error: string | null; attempts: number; created_at: number; updated_at: number; +} + +const OUTPUT_CAP = 8_192; +const START_BATCH = 5; // pending examples started per pass +const ADVANCE_BUDGET_MS = 25_000; +const MAX_ATTEMPTS = 4; + +const json = (body: unknown, status = 200): Response => + new Response(JSON.stringify(body), { status, headers: { "content-type": "application/json" } }); +const now = () => Math.floor(Date.now() / 1000); +const rid = (prefix: string): string => { + const b = crypto.getRandomValues(new Uint8Array(12)); + return prefix + "_" + [...b].map((x) => x.toString(16).padStart(2, "0")).join(""); +}; +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); + +// ── router ─────────────────────────────────────────────────────────────── +export async function handleEvals( + req: Request, env: DashboardEnv, caller: EvalCaller, ctx: ExecutionContext, sub: string, method: string, +): Promise { + let m: RegExpMatchArray | null; + + // dataset collection (agent-scoped via ?agent_id= on GET, body.agent_id on POST) + if (sub === "/evals") { + if (method === "GET") return listDatasets(req, env, caller); + if (method === "POST") return createDataset(req, env, caller); + } + // per-run (order before the dataset pattern so "runs" isn't read as a dataset id) + if ((m = sub.match(/^\/evals\/runs\/([^/]+)\/results$/)) && method === "GET") return listResults(env, caller, m[1]); + if ((m = sub.match(/^\/evals\/runs\/([^/]+)$/)) && method === "GET") return getRun(env, caller, m[1]); + // per-dataset runs + if ((m = sub.match(/^\/evals\/([^/]+)\/runs$/))) { + if (method === "GET") return listRuns(env, caller, m[1]); + if (method === "POST") return createRun(env, caller, ctx, m[1]); + } + // single dataset + if ((m = sub.match(/^\/evals\/([^/]+)$/))) { + if (method === "GET") return getDataset(env, caller, m[1]); + if (method === "PATCH") return updateDataset(req, env, caller, m[1]); + if (method === "DELETE") return deleteDataset(env, caller, m[1]); + } + return json({ error: "not found" }, 404); +} + +// ── datasets ─────────────────────────────────────────────────────────────── +async function listDatasets(req: Request, env: DashboardEnv, caller: EvalCaller): Promise { + const agentId = new URL(req.url).searchParams.get("agent_id"); + if (!agentId) return json({ error: "agent_id is required" }, 400); + const { results } = await env.OPENCOMPUTER_DB.prepare( + `SELECT * FROM eval_datasets WHERE org_id = ?1 AND agent_id = ?2 ORDER BY updated_at DESC LIMIT 200`, + ).bind(caller.orgID, agentId).all(); + return json({ data: (results ?? []).map(serializeDataset) }); +} + +async function getDataset(env: DashboardEnv, caller: EvalCaller, id: string): Promise { + const row = await loadDataset(env, caller.orgID, id); + if (!row) return json({ error: "dataset not found" }, 404); + return json(serializeDataset(row)); +} + +async function createDataset(req: Request, env: DashboardEnv, caller: EvalCaller): Promise { + const body = (await req.json().catch(() => ({}))) as { name?: string; agent_id?: string; examples?: unknown }; + const agentId = typeof body.agent_id === "string" ? body.agent_id.trim() : ""; + if (!agentId) return json({ error: "agent_id is required" }, 400); + const name = typeof body.name === "string" && body.name.trim() ? body.name.trim() : "Untitled"; + const examples = normalizeExamples(body.examples); + if (examples.error) return json({ error: examples.error }, 400); + const id = rid("evd"), ts = now(); + await env.OPENCOMPUTER_DB.prepare( + `INSERT INTO eval_datasets (id, org_id, agent_id, name, examples, created_at, updated_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?6)`, + ).bind(id, caller.orgID, agentId, name, JSON.stringify(examples.list), ts).run(); + const row = await loadDataset(env, caller.orgID, id); + return json(serializeDataset(row!), 201); +} + +async function updateDataset(req: Request, env: DashboardEnv, caller: EvalCaller, id: string): Promise { + const row = await loadDataset(env, caller.orgID, id); + if (!row) return json({ error: "dataset not found" }, 404); + const body = (await req.json().catch(() => ({}))) as { name?: string; examples?: unknown }; + const name = typeof body.name === "string" && body.name.trim() ? body.name.trim() : row.name; + let examplesJson = row.examples; + if (body.examples !== undefined) { + const examples = normalizeExamples(body.examples); + if (examples.error) return json({ error: examples.error }, 400); + examplesJson = JSON.stringify(examples.list); + } + await env.OPENCOMPUTER_DB.prepare( + `UPDATE eval_datasets SET name = ?1, examples = ?2, updated_at = ?3 WHERE id = ?4 AND org_id = ?5`, + ).bind(name, examplesJson, now(), id, caller.orgID).run(); + const updated = await loadDataset(env, caller.orgID, id); + return json(serializeDataset(updated!)); +} + +async function deleteDataset(env: DashboardEnv, caller: EvalCaller, id: string): Promise { + await env.OPENCOMPUTER_DB.prepare(`DELETE FROM eval_datasets WHERE id = ?1 AND org_id = ?2`) + .bind(id, caller.orgID).run(); + return new Response(null, { status: 204 }); +} + +// ── runs ───────────────────────────────────────────────────────────────── +async function listRuns(env: DashboardEnv, caller: EvalCaller, datasetId: string): Promise { + const { results } = await env.OPENCOMPUTER_DB.prepare( + `SELECT * FROM eval_runs WHERE org_id = ?1 AND dataset_id = ?2 ORDER BY created_at DESC LIMIT 50`, + ).bind(caller.orgID, datasetId).all(); + return json({ data: (results ?? []).map(serializeRun) }); +} + +async function getRun(env: DashboardEnv, caller: EvalCaller, runId: string): Promise { + const run = await env.OPENCOMPUTER_DB.prepare(`SELECT * FROM eval_runs WHERE id = ?1 AND org_id = ?2`) + .bind(runId, caller.orgID).first(); + if (!run) return json({ error: "run not found" }, 404); + return json(serializeRun(run)); +} + +async function listResults(env: DashboardEnv, caller: EvalCaller, runId: string): Promise { + const { results } = await env.OPENCOMPUTER_DB.prepare( + `SELECT * FROM eval_results WHERE run_id = ?1 AND org_id = ?2 ORDER BY created_at LIMIT 1000`, + ).bind(runId, caller.orgID).all(); + return json({ data: (results ?? []).map(serializeResult) }); +} + +async function createRun(env: DashboardEnv, caller: EvalCaller, ctx: ExecutionContext, datasetId: string): Promise { + const dataset = await loadDataset(env, caller.orgID, datasetId); + if (!dataset) return json({ error: "dataset not found" }, 404); + const examples = JSON.parse(dataset.examples) as Example[]; + if (examples.length === 0) return json({ error: "dataset has no examples" }, 400); + + const runId = rid("evr"), ts = now(); + await env.OPENCOMPUTER_DB.prepare( + `INSERT INTO eval_runs (id, org_id, dataset_id, agent_id, status, total, created_at, updated_at) + VALUES (?1, ?2, ?3, ?4, 'pending', ?5, ?6, ?6)`, + ).bind(runId, caller.orgID, datasetId, dataset.agent_id, examples.length, ts).run(); + + // One result row per example (batched insert). + const stmts = examples.map((ex) => + env.OPENCOMPUTER_DB.prepare( + `INSERT INTO eval_results (id, run_id, org_id, example_id, input, expect, state, created_at, updated_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, 'pending', ?7, ?7)`, + ).bind(rid("evres"), runId, caller.orgID, ex.id, ex.input, JSON.stringify(ex.expect ?? {}), ts), + ); + await env.OPENCOMPUTER_DB.batch(stmts); + + // Kick the runner now (fast path for small sets); the cron reconciles the rest. + ctx.waitUntil(advanceRun(env, runId).catch((e) => console.error("evals: advanceRun failed", runId, e))); + const run = await env.OPENCOMPUTER_DB.prepare(`SELECT * FROM eval_runs WHERE id = ?1`).bind(runId).first(); + return json(serializeRun(run!), 201); +} + +// ── runner (resumable state machine) ───────────────────────────────────────── +export async function advanceRun(env: DashboardEnv, runId: string, budgetMs = ADVANCE_BUDGET_MS): Promise { + const start = Date.now(); + const run = await env.OPENCOMPUTER_DB.prepare(`SELECT * FROM eval_runs WHERE id = ?1`).bind(runId).first(); + if (!run || run.status === "done" || run.status === "failed") return; + if (!env.OC_ORG_TOKEN_SECRET) { await failRun(env, runId, "agent sessions not configured"); return; } + if (run.status === "pending") { + await env.OPENCOMPUTER_DB.prepare(`UPDATE eval_runs SET status = 'running', updated_at = ?2 WHERE id = ?1`) + .bind(runId, now()).run(); + } + + while (Date.now() - start < budgetMs) { + const { results } = await env.OPENCOMPUTER_DB.prepare( + `SELECT * FROM eval_results WHERE run_id = ?1 ORDER BY created_at`, + ).bind(runId).all(); + const rows = results ?? []; + const pending = rows.filter((r) => r.state === "pending"); + const running = rows.filter((r) => r.state === "running"); + if (pending.length === 0 && running.length === 0) break; // all terminal + + await Promise.all(pending.slice(0, START_BATCH).map((r) => startExample(env, run, r))); + await Promise.all(running.map((r) => pollExample(env, run, r))); + await updateRunAggregate(env, runId); + + // Everything in flight and nothing to start → let sessions make progress before re-polling. + if (pending.length <= START_BATCH && running.length > 0) await sleep(1500); + } + await updateRunAggregate(env, runId); +} + +async function startExample(env: DashboardEnv, run: RunRow, row: ResultRow): Promise { + try { + const res = await sessionsFetch(env, run.org_id, "POST", "/v3/sessions", { + agent: run.agent_id, input: { text: row.input }, metadata: { eval_run: run.id }, + }); + if (!res.ok) throw new Error(`create session HTTP ${res.status}: ${(await res.text()).slice(0, 200)}`); + const session = (await res.json()) as { id?: string; session?: { id?: string } }; + const sessionId = session.id ?? session.session?.id; + if (!sessionId) throw new Error("create session: no id in response"); + await env.OPENCOMPUTER_DB.prepare( + `UPDATE eval_results SET state = 'running', session_id = ?2, attempts = attempts + 1, updated_at = ?3 WHERE id = ?1`, + ).bind(row.id, sessionId, now()).run(); + } catch (e) { + await bumpOrFail(env, row, e); + } +} + +async function pollExample(env: DashboardEnv, run: RunRow, row: ResultRow): Promise { + if (!row.session_id) return; + try { + const res = await sessionsFetch(env, run.org_id, "GET", `/v3/sessions/${encodeURIComponent(row.session_id)}/result`); + if (!res.ok) throw new Error(`result HTTP ${res.status}`); + const body = (await res.json()) as { last_turn?: { state?: string; usage?: Record }; result?: unknown }; + const state = body.last_turn?.state; + if (!state || state === "running") return; // still working + const output = extractText(body.result).slice(0, OUTPUT_CAP); + const usage = body.last_turn?.usage ?? {}; + const cost = numOf(usage.cost_usd) ?? numOf((usage.cost as Record | undefined)?.total); + const tokens = numOf(usage.total_tokens); + const expect = JSON.parse(row.expect) as Expect; + let tools: string[] = []; + if (expect.tools && expect.tools.length) tools = await fetchToolsCalled(env, run.org_id, row.session_id); + const scores = scoreOutput(expect, { output, outcome: state, cost_usd: cost, tools }); + const passed = scores.every((s) => s.pass) ? 1 : 0; + await env.OPENCOMPUTER_DB.prepare( + `UPDATE eval_results SET state = 'done', output = ?2, outcome = ?3, cost_usd = ?4, tokens = ?5, + scores = ?6, passed = ?7, updated_at = ?8 WHERE id = ?1`, + ).bind(row.id, output, state, cost ?? null, tokens ?? null, JSON.stringify(scores), passed, now()).run(); + } catch (e) { + await bumpOrFail(env, row, e); + } +} + +async function bumpOrFail(env: DashboardEnv, row: ResultRow, e: unknown): Promise { + const attempts = row.attempts + 1; + const msg = e instanceof Error ? e.message : String(e); + if (attempts >= MAX_ATTEMPTS) { + await env.OPENCOMPUTER_DB.prepare( + `UPDATE eval_results SET state = 'failed', attempts = ?2, error = ?3, updated_at = ?4 WHERE id = ?1`, + ).bind(row.id, attempts, msg.slice(0, 500), now()).run(); + } else { + await env.OPENCOMPUTER_DB.prepare( + `UPDATE eval_results SET attempts = ?2, error = ?3, updated_at = ?4 WHERE id = ?1`, + ).bind(row.id, attempts, msg.slice(0, 500), now()).run(); + } +} + +async function updateRunAggregate(env: DashboardEnv, runId: string): Promise { + const agg = await env.OPENCOMPUTER_DB.prepare( + `SELECT COUNT(*) AS total, + SUM(CASE WHEN state IN ('done','failed') THEN 1 ELSE 0 END) AS completed, + SUM(CASE WHEN passed = 1 THEN 1 ELSE 0 END) AS passed + FROM eval_results WHERE run_id = ?1`, + ).bind(runId).first<{ total: number; completed: number; passed: number }>(); + const total = agg?.total ?? 0, completed = agg?.completed ?? 0, passed = agg?.passed ?? 0; + const allDone = total > 0 && completed >= total; + const score = completed > 0 ? passed / completed : null; + await env.OPENCOMPUTER_DB.prepare( + `UPDATE eval_runs SET completed = ?2, passed = ?3, score = ?4, + status = CASE WHEN ?5 = 1 THEN 'done' ELSE status END, + finished_at = CASE WHEN ?5 = 1 THEN ?6 ELSE finished_at END, + updated_at = ?6 WHERE id = ?1`, + ).bind(runId, completed, passed, score, allDone ? 1 : 0, now()).run(); +} + +async function failRun(env: DashboardEnv, runId: string, error: string): Promise { + await env.OPENCOMPUTER_DB.prepare( + `UPDATE eval_runs SET status = 'failed', error = ?2, finished_at = ?3, updated_at = ?3 WHERE id = ?1`, + ).bind(runId, error, now()).run(); +} + +// Cron reconciler: resume any non-terminal run whose rows still have work (5-min tick backstop). +export async function reconcileEvalRuns(env: DashboardEnv): Promise { + if (!env.OC_ORG_TOKEN_SECRET) return; + const { results } = await env.OPENCOMPUTER_DB.prepare( + `SELECT id FROM eval_runs WHERE status IN ('pending','running') ORDER BY updated_at LIMIT 10`, + ).all<{ id: string }>(); + for (const r of results ?? []) { + await advanceRun(env, r.id).catch((e) => console.error("evals: reconcile failed", r.id, e)); + } +} + +// ── sessions-api access (act for the org) ──────────────────────────────────── +async function sessionsFetch( + env: DashboardEnv, orgId: string, method: string, path: string, body?: unknown, +): Promise { + const base = (env.SESSIONS_API_URL ?? "https://api.opencomputer.dev").replace(/\/+$/, ""); + const token = await mintOrgToken(env.OC_ORG_TOKEN_SECRET!, orgId, null); + const headers: Record = { "x-oc-org-token": token }; + const init: RequestInit = { method, headers }; + if (body !== undefined) { headers["content-type"] = "application/json"; init.body = JSON.stringify(body); } + return fetch(base + path, init); +} + +async function fetchToolsCalled(env: DashboardEnv, orgId: string, sessionId: string): Promise { + try { + const res = await sessionsFetch(env, orgId, "GET", `/v3/sessions/${encodeURIComponent(sessionId)}/events?limit=500`); + if (!res.ok) return []; + const body = (await res.json()) as { data?: Array<{ type?: string; body?: { tool?: string } }> }; + return (body.data ?? []).filter((e) => e.type === "tool.call").map((e) => e.body?.tool ?? "").filter(Boolean); + } catch { return []; } +} + +// ── scoring (deterministic; LLM-judge plugs in here later) ─────────────────── +export function scoreOutput( + expect: Expect, obs: { output: string; outcome: string; cost_usd?: number; tools: string[] }, +): Score[] { + const scores: Score[] = []; + const out = obs.output ?? ""; + const norm = (s: string) => s.trim().toLowerCase(); + + scores.push({ + name: "completed", + pass: obs.outcome !== "error" && obs.outcome !== "failed", + detail: `turn outcome: ${obs.outcome}`, + }); + if (expect.contains?.length) { + const missing = expect.contains.filter((s) => !out.toLowerCase().includes(s.toLowerCase())); + scores.push({ name: "contains", pass: missing.length === 0, detail: missing.length ? `missing: ${missing.join(", ")}` : undefined }); + } + if (expect.equals !== undefined) { + scores.push({ name: "equals", pass: norm(out) === norm(expect.equals) }); + } + if (expect.iregex !== undefined) { + let pass = false, detail: string | undefined; + try { pass = new RegExp(expect.iregex, "i").test(out); } catch { detail = "invalid regex"; } + scores.push({ name: "regex", pass, detail }); + } + if (expect.outcome !== undefined) { + scores.push({ name: "outcome", pass: obs.outcome === expect.outcome, detail: `got ${obs.outcome}` }); + } + if (expect.tools?.length) { + const missing = expect.tools.filter((t) => !obs.tools.includes(t)); + scores.push({ name: "tools", pass: missing.length === 0, detail: missing.length ? `not called: ${missing.join(", ")}` : undefined }); + } + if (expect.max_cost_usd !== undefined) { + const cost = obs.cost_usd ?? 0; + scores.push({ name: "cost", pass: cost <= expect.max_cost_usd, detail: `$${cost.toFixed(4)} ≤ $${expect.max_cost_usd}` }); + } + return scores; +} + +// ── helpers ────────────────────────────────────────────────────────────── +function loadDataset(env: DashboardEnv, orgId: string, id: string): Promise { + return env.OPENCOMPUTER_DB.prepare(`SELECT * FROM eval_datasets WHERE id = ?1 AND org_id = ?2`) + .bind(id, orgId).first(); +} + +function normalizeExamples(raw: unknown): { list: Example[]; error?: string } { + if (raw === undefined) return { list: [] }; + if (!Array.isArray(raw)) return { list: [], error: "examples must be an array" }; + if (raw.length > 500) return { list: [], error: "too many examples (max 500)" }; + const list: Example[] = []; + for (const e of raw) { + const ex = e as { id?: unknown; input?: unknown; expect?: unknown }; + if (typeof ex.input !== "string" || !ex.input.trim()) return { list: [], error: "each example needs a non-empty input" }; + list.push({ + id: typeof ex.id === "string" && ex.id ? ex.id : rid("ex"), + input: ex.input, + ...(ex.expect && typeof ex.expect === "object" ? { expect: ex.expect as Expect } : {}), + }); + } + return { list }; +} + +function extractText(result: unknown): string { + const body = (result as { body?: unknown } | null | undefined)?.body; + if (body == null) return ""; + if (typeof body === "string") return body; + const text = (body as { text?: unknown }).text; + if (typeof text === "string") return text; + return JSON.stringify(body); +} +function numOf(v: unknown): number | undefined { return typeof v === "number" && Number.isFinite(v) ? v : undefined; } + +function serializeDataset(r: DatasetRow) { + return { + id: r.id, agent_id: r.agent_id, name: r.name, + examples: JSON.parse(r.examples) as Example[], + created_at: r.created_at, updated_at: r.updated_at, + }; +} +function serializeRun(r: RunRow) { + return { + id: r.id, dataset_id: r.dataset_id, agent_id: r.agent_id, status: r.status, + total: r.total, completed: r.completed, passed: r.passed, score: r.score, error: r.error, + created_at: r.created_at, updated_at: r.updated_at, finished_at: r.finished_at, + }; +} +function serializeResult(r: ResultRow) { + return { + id: r.id, run_id: r.run_id, example_id: r.example_id, input: r.input, + expect: JSON.parse(r.expect) as Expect, state: r.state, session_id: r.session_id, + output: r.output, outcome: r.outcome, cost_usd: r.cost_usd, tokens: r.tokens, + scores: JSON.parse(r.scores) as Score[], passed: r.passed, error: r.error, + created_at: r.created_at, updated_at: r.updated_at, + }; +} diff --git a/cloudflare-workers/api-edge/src/index.ts b/cloudflare-workers/api-edge/src/index.ts index a3cc23c2..d8169338 100644 --- a/cloudflare-workers/api-edge/src/index.ts +++ b/cloudflare-workers/api-edge/src/index.ts @@ -21,6 +21,7 @@ export { CreditAccount } from "../../shared/credit_account"; import { handleDashboard, type DashboardEnv } from "./dashboard"; +import { reconcileEvalRuns } from "./evals"; import { AGENT_SECURITY_NOTIFICATION_PATH, receiveAgentSecurityNotification, @@ -2953,6 +2954,10 @@ export default { ctx.waitUntil( runPausedCapEnforcer(env).catch((err) => console.error("paused-cap: run failed", err)), ); + // Resume any in-flight eval runs (backstop for the request-time kick). Billing-independent. + ctx.waitUntil( + reconcileEvalRuns(env).catch((err) => console.error("evals: reconcile failed", err)), + ); if (!env.AUTUMN_SECRET_KEY) return; ctx.waitUntil( runAutumnMeter(env, Date.now()).catch((err) => console.error("autumn-meter: run failed", err)), diff --git a/web/src/api/evals.ts b/web/src/api/evals.ts new file mode 100644 index 00000000..502445de --- /dev/null +++ b/web/src/api/evals.ts @@ -0,0 +1,103 @@ +// Native agent evals — edge-owned (D1 in api-edge), not a /v3 sessions-api resource. +// The runner drives the session API to run the agent, so this is runtime-agnostic: +// flue, langgraph, claude, codex and pi are all just "the agent under test". +import { apiFetch } from './client' + +export interface EvalExpect { + contains?: string[] + equals?: string + iregex?: string + outcome?: string + tools?: string[] + max_cost_usd?: number +} +export interface EvalExample { + id: string + input: string + expect?: EvalExpect +} +export interface EvalDataset { + id: string + agent_id: string + name: string + examples: EvalExample[] + created_at: number + updated_at: number +} +export type EvalRunStatus = 'pending' | 'running' | 'done' | 'failed' +export interface EvalRun { + id: string + dataset_id: string + agent_id: string + status: EvalRunStatus + total: number + completed: number + passed: number + score: number | null + error: string | null + created_at: number + updated_at: number + finished_at: number | null +} +export interface EvalScore { + name: string + pass: boolean + detail?: string +} +export interface EvalResult { + id: string + run_id: string + example_id: string + input: string + expect: EvalExpect + state: 'pending' | 'running' | 'done' | 'failed' + session_id: string | null + output: string | null + outcome: string | null + cost_usd: number | null + tokens: number | null + scores: EvalScore[] + passed: number | null + error: string | null +} + +export const getEvalDatasets = (agentId: string) => + apiFetch<{ data: EvalDataset[] }>( + `/evals?agent_id=${encodeURIComponent(agentId)}`, + ).then((r) => r.data) + +export const createEvalDataset = (body: { + agent_id: string + name: string + examples: EvalExample[] +}) => + apiFetch('/evals', { + method: 'POST', + body: JSON.stringify(body), + }) + +export const updateEvalDataset = ( + id: string, + body: { name?: string; examples?: EvalExample[] }, +) => + apiFetch(`/evals/${id}`, { + method: 'PATCH', + body: JSON.stringify(body), + }) + +export const deleteEvalDataset = (id: string) => + apiFetch(`/evals/${id}`, { method: 'DELETE' }) + +export const getEvalRuns = (datasetId: string) => + apiFetch<{ data: EvalRun[] }>(`/evals/${datasetId}/runs`).then((r) => r.data) + +export const createEvalRun = (datasetId: string) => + apiFetch(`/evals/${datasetId}/runs`, { + method: 'POST', + body: JSON.stringify({}), + }) + +export const getEvalResults = (runId: string) => + apiFetch<{ data: EvalResult[] }>(`/evals/runs/${runId}/results`).then( + (r) => r.data, + ) diff --git a/web/src/components/agent-evals.test.tsx b/web/src/components/agent-evals.test.tsx new file mode 100644 index 00000000..e9e03e8f --- /dev/null +++ b/web/src/components/agent-evals.test.tsx @@ -0,0 +1,39 @@ +import { QueryClient, QueryClientProvider } from '@tanstack/react-query' +import { renderToStaticMarkup } from 'react-dom/server' +import { MemoryRouter } from 'react-router-dom' +import { describe, expect, it } from 'vitest' +import { AgentEvals } from './agent-evals' + +const agentId = 'agt_0123456789abcdef01234567' + +function render() { + const qc = new QueryClient({ + defaultOptions: { queries: { retry: false, staleTime: Infinity } }, + }) + qc.setQueryData(['eval-datasets', agentId], [ + { + id: 'evd_1', + agent_id: agentId, + name: 'smoke suite', + examples: [{ id: 'ex1', input: 'What is 2+2?', expect: { contains: ['4'] } }], + created_at: 1, + updated_at: 1, + }, + ]) + return renderToStaticMarkup( + + + + + , + ) +} + +describe('AgentEvals', () => { + it('renders the datasets for an agent', () => { + const markup = render() + expect(markup).toContain('smoke suite') + expect(markup).toContain('Evals') + expect(markup).toContain('New dataset') + }) +}) diff --git a/web/src/components/agent-evals.tsx b/web/src/components/agent-evals.tsx new file mode 100644 index 00000000..70295f23 --- /dev/null +++ b/web/src/components/agent-evals.tsx @@ -0,0 +1,448 @@ +// Agent Evals tab — native input/output evals for a deployed agent. A dataset is a set of +// { input, expect } examples; a run executes each as an isolated session against the agent and +// scores the output. Runtime-agnostic (drives the /v3 session API), so flue/langgraph/claude/ +// codex/pi are all just "the agent under test". Backed by api-edge + D1 (see src/api/evals.ts). +import { useState } from 'react' +import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query' +import { FlaskConical, Play, Plus, Trash2, ChevronLeft, Pencil } from 'lucide-react' +import { + Panel, + PanelHeader, + PanelTitle, + PanelDescription, + PanelContent, + PanelFooter, +} from '@/components/panel' +import { Button } from '@/components/ui/button' +import { Field, Input, Textarea } from '@/components/form' +import { StatusBadge } from '@/components/status-badge' +import { EmptyState } from '@/components/empty-state' +import { ResourceTable, type Column } from '@/components/resource-table' +import { notifyError } from '@/lib/errors' +import { + createEvalDataset, + createEvalRun, + deleteEvalDataset, + getEvalDatasets, + getEvalResults, + getEvalRuns, + updateEvalDataset, + type EvalDataset, + type EvalExample, + type EvalExpect, + type EvalResult, + type EvalRun, +} from '@/api/evals' + +const EXAMPLES_PLACEHOLDER = `[ + { "input": "What is 2 + 2?", "expect": { "contains": ["4"] } }, + { "input": "Capital of France?", "expect": { "contains": ["Paris"], "max_cost_usd": 0.05 } } +]` + +const runBadge = (s: EvalRun['status']) => + s === 'done' ? 'success' : s === 'failed' ? 'error' : s === 'running' ? 'running' : 'pending' + +/** Human-readable summary of an example's checks. */ +function expectSummary(e?: EvalExpect): string { + if (!e) return 'no checks' + const parts: string[] = [] + if (e.contains?.length) parts.push(`contains ${e.contains.map((s) => JSON.stringify(s)).join(', ')}`) + if (e.equals !== undefined) parts.push(`equals ${JSON.stringify(e.equals)}`) + if (e.iregex !== undefined) parts.push(`regex /${e.iregex}/i`) + if (e.outcome !== undefined) parts.push(`outcome=${e.outcome}`) + if (e.tools?.length) parts.push(`tools ${e.tools.join(', ')}`) + if (e.max_cost_usd !== undefined) parts.push(`cost ≤ $${e.max_cost_usd}`) + return parts.length ? parts.join(' · ') : 'no checks' +} + +export function AgentEvals({ agentId }: { agentId: string }) { + const [selectedDataset, setSelectedDataset] = useState(null) + const [selectedRun, setSelectedRun] = useState(null) + const [creating, setCreating] = useState(false) + + const datasetsQuery = useQuery({ + queryKey: ['eval-datasets', agentId], + queryFn: () => getEvalDatasets(agentId), + }) + const datasets = datasetsQuery.data ?? [] + + if (selectedRun && selectedDataset) { + return setSelectedRun(null)} /> + } + if (selectedDataset) { + return ( + d.id === selectedDataset)} + datasetId={selectedDataset} + onBack={() => setSelectedDataset(null)} + onOpenRun={(id) => setSelectedRun(id)} + /> + ) + } + + return ( + + +
+ Evals + + Input/output eval sets for this agent. Each run executes every example as an isolated + session and scores the output — the same for any runtime. + +
+ + + + {creating ? ( + setCreating(false)} + onCancel={() => setCreating(false)} + /> + ) : null} + + + columns={datasetColumns(setSelectedDataset)} + rows={datasets} + rowKey={(d) => d.id} + loading={datasetsQuery.isLoading} + empty={ + + } + /> + + ) +} + +function datasetColumns(select: (id: string) => void): Column[] { + return [ + { + key: 'name', + header: 'Dataset', + cell: (d) => ( + + ), + }, + { + key: 'examples', + header: 'Examples', + cell: (d) => {d.examples.length}, + }, + { + key: 'open', + header: '', + cell: (d) => ( + + ), + }, + ] +} + +/** Create OR edit a dataset. `dataset` present → edit mode (pre-filled, PATCH); else create. */ +function DatasetForm({ + agentId, + dataset, + onDone, + onCancel, +}: { + agentId: string + dataset?: EvalDataset + onDone: () => void + onCancel: () => void +}) { + const queryClient = useQueryClient() + const [name, setName] = useState(dataset?.name ?? '') + const [text, setText] = useState( + dataset ? JSON.stringify(dataset.examples.map((e) => ({ input: e.input, expect: e.expect })), null, 2) : '', + ) + const [parseError, setParseError] = useState(null) + + const invalidate = () => + void queryClient.invalidateQueries({ queryKey: ['eval-datasets', agentId] }) + const mutation = useMutation({ + mutationFn: (examples: EvalExample[]) => + dataset + ? updateEvalDataset(dataset.id, { name: name.trim() || 'Untitled', examples }) + : createEvalDataset({ agent_id: agentId, name: name.trim() || 'Untitled', examples }), + onSuccess: () => { + invalidate() + onDone() + }, + onError: (e) => notifyError(dataset ? "Couldn't save the dataset." : "Couldn't create the dataset.", e), + }) + + const submit = () => { + setParseError(null) + let examples: { input: string; expect?: unknown }[] + try { + const parsed: unknown = JSON.parse(text || '[]') + if (!Array.isArray(parsed)) throw new Error('examples must be a JSON array') + examples = parsed as { input: string; expect?: unknown }[] + if (!examples.every((e) => typeof e?.input === 'string' && e.input.trim())) { + throw new Error('each example needs a non-empty "input"') + } + } catch (err) { + setParseError(err instanceof Error ? err.message : 'invalid JSON') + return + } + mutation.mutate( + examples.map((e) => ({ + id: crypto.randomUUID(), + input: e.input, + ...(e.expect && typeof e.expect === 'object' ? { expect: e.expect as EvalExpect } : {}), + })), + ) + } + + return ( + +
+ + setName(e.target.value)} + placeholder="e.g. smoke suite" + /> + + +