native input/output agent evals

cloudflare-workers/api-edge/migrations/0004_evals.sql

@@ -0,0 +1,61 @@
+-- Native agent evals (input/output). A dataset is a set of {input, expect} examples for one
+-- deployed agent; a run executes every example as an isolated session against that agent and
+-- scores the output. Runtime-agnostic: the runner drives the /v3 session API, so flue, langgraph,
+-- claude, codex and pi are all just "the agent under test". Owned by api-edge + D1 (a consumer of
+-- sessions-api, not part of it).
+
+-- A reusable set of {input, expect} examples for a single agent.
+CREATE TABLE IF NOT EXISTS eval_datasets (
+  id          TEXT PRIMARY KEY,            -- evd_<hex>
+  org_id      TEXT NOT NULL,
+  agent_id    TEXT NOT NULL,               -- the sessions-api agent id (system under test)
+  name        TEXT NOT NULL,
+  examples    TEXT NOT NULL DEFAULT '[]',  -- JSON: [{ id, input, expect?: {contains?,equals?,iregex?,tools?,outcome?,max_cost_usd?} }]
+  created_at  INTEGER NOT NULL,
+  updated_at  INTEGER NOT NULL
+);
+CREATE INDEX IF NOT EXISTS idx_eval_datasets_org_agent ON eval_datasets (org_id, agent_id, updated_at DESC);
+
+-- One execution of a dataset against the agent (pinned to the live revision at run time).
+CREATE TABLE IF NOT EXISTS eval_runs (
+  id           TEXT PRIMARY KEY,           -- evr_<hex>
+  org_id       TEXT NOT NULL,
+  dataset_id   TEXT NOT NULL,
+  agent_id     TEXT NOT NULL,
+  status       TEXT NOT NULL DEFAULT 'pending',  -- pending | running | done | failed
+  total        INTEGER NOT NULL DEFAULT 0,
+  completed    INTEGER NOT NULL DEFAULT 0,        -- results in a terminal state
+  passed       INTEGER NOT NULL DEFAULT 0,        -- results with every check passing
+  score        REAL,                              -- passed / completed (0..1), null until any complete
+  error        TEXT,
+  created_at   INTEGER NOT NULL,
+  updated_at   INTEGER NOT NULL,
+  finished_at  INTEGER
+);
+CREATE INDEX IF NOT EXISTS idx_eval_runs_dataset ON eval_runs (dataset_id, created_at DESC);
+CREATE INDEX IF NOT EXISTS idx_eval_runs_status ON eval_runs (status, updated_at);
+
+-- Per-example result within a run. Drives a small state machine (pending -> running -> done/failed)
+-- so the cron + a request-time kick can resume it without a Durable Object.
+CREATE TABLE IF NOT EXISTS eval_results (
+  id          TEXT PRIMARY KEY,            -- evres_<hex>
+  run_id      TEXT NOT NULL,
+  org_id      TEXT NOT NULL,
+  example_id  TEXT NOT NULL,
+  input       TEXT NOT NULL,               -- denormalized example input
+  expect      TEXT NOT NULL DEFAULT '{}',  -- denormalized example expectations (JSON)
+  state       TEXT NOT NULL DEFAULT 'pending',   -- pending | running | done | failed
+  session_id  TEXT,                        -- the /v3 session created for this example
+  output      TEXT,                        -- final agent message text
+  outcome     TEXT,                        -- terminal turn state (ok | error | ...)
+  cost_usd    REAL,
+  tokens      INTEGER,
+  scores      TEXT NOT NULL DEFAULT '[]',  -- JSON: [{ name, pass, detail }]
+  passed      INTEGER,                     -- 0/1: every check passed
+  error       TEXT,
+  attempts    INTEGER NOT NULL DEFAULT 0,
+  created_at  INTEGER NOT NULL,
+  updated_at  INTEGER NOT NULL
+);
+CREATE INDEX IF NOT EXISTS idx_eval_results_run ON eval_results (run_id, created_at);
+CREATE INDEX IF NOT EXISTS idx_eval_results_state ON eval_results (state, updated_at);

cloudflare-workers/api-edge/src/dashboard.ts

@@ -31,6 +31,7 @@ import {
   acknowledgeAgentSecurityNotification,
   listAgentSecurityNotifications,
 } from "./agent_security_notifications";
+import { handleEvals } from "./evals";
 import { createAPIKey } from "./api_keys";
 
 export interface DashboardEnv {
@@ -157,7 +158,7 @@ async function mintCellCapToken(secret: string, orgID: string, cellID: string, p
 // OC_ORG_TOKEN_SECRET (shared with sessions-api). /v3 trusts it and sets owner =
 // the asserted org — same "act for org X" shape as the cell cap-token, so no
 // osb_ key reaches the browser and /v3 never custodies a customer key.
-async function mintOrgToken(secret: string, orgID: string, userID: string | null): Promise<string> {
+export async function mintOrgToken(secret: string, orgID: string, userID: string | null): Promise<string> {
   const now = Math.floor(Date.now() / 1000);
   const header = { alg: "HS256", typ: "JWT" };
   const payload: Record<string, unknown> = {
@@ -889,7 +890,7 @@ async function proxyToBrowserAPI(
 export async function handleDashboard(
   req: Request,
   env: DashboardEnv,
-  _ctx: ExecutionContext,
+  ctx: ExecutionContext,
   path: string,
 ): Promise<Response> {
   const caller = await authDashboard(req, env);
@@ -907,6 +908,11 @@ export async function handleDashboard(
     return proxyToV3(req, env, caller, sub);
   }
 
+  // ── Evals (edge-owned, D1-backed; the runner drives /v3 to run the agent) ──
+  if (sub === "/evals" || sub.startsWith("/evals/")) {
+    return handleEvals(req, env, caller, ctx, sub, method);
+  }
+
   // ── Browser Sessions — proxy to the dedicated browser Worker ───────────
   if (sub === "/browsers" && method === "GET") return proxyToBrowserAPI(req, env, caller, "/v1/browsers");
   if (sub === "/browser-usage" && method === "GET") return proxyToBrowserAPI(req, env, caller, "/v1/browser-usage");

cloudflare-workers/api-edge/src/evals.test.ts

@@ -0,0 +1,46 @@
+import { describe, expect, it } from "vitest";
+import { scoreOutput } from "./evals";
+
+const pass = (scores: { name: string; pass: boolean }[], name: string) =>
+  scores.find((s) => s.name === name)?.pass;
+
+describe("evals scoreOutput", () => {
+  it("always scores completion from the turn outcome", () => {
+    expect(pass(scoreOutput({}, { output: "hi", outcome: "ok", tools: [] }), "completed")).toBe(true);
+    expect(pass(scoreOutput({}, { output: "", outcome: "error", tools: [] }), "completed")).toBe(false);
+  });
+
+  it("contains: case-insensitive, all substrings required", () => {
+    const obs = { output: "Paris is the capital, and 12*13 = 156.", outcome: "ok", tools: [] };
+    expect(pass(scoreOutput({ contains: ["paris", "156"] }, obs), "contains")).toBe(true);
+    expect(pass(scoreOutput({ contains: ["paris", "999"] }, obs), "contains")).toBe(false);
+  });
+
+  it("equals: normalized (trim + case)", () => {
+    expect(pass(scoreOutput({ equals: "yes" }, { output: "  YES ", outcome: "ok", tools: [] }), "equals")).toBe(true);
+    expect(pass(scoreOutput({ equals: "yes" }, { output: "no", outcome: "ok", tools: [] }), "equals")).toBe(false);
+  });
+
+  it("regex: case-insensitive; invalid regex fails safe", () => {
+    expect(pass(scoreOutput({ iregex: "^\\d+$" }, { output: "42", outcome: "ok", tools: [] }), "regex")).toBe(true);
+    const bad = scoreOutput({ iregex: "(" }, { output: "x", outcome: "ok", tools: [] });
+    expect(pass(bad, "regex")).toBe(false);
+    expect(bad.find((s) => s.name === "regex")?.detail).toBe("invalid regex");
+  });
+
+  it("tools: every named tool must be called", () => {
+    const obs = { output: "done", outcome: "ok", tools: ["bash", "read"] };
+    expect(pass(scoreOutput({ tools: ["bash"] }, obs), "tools")).toBe(true);
+    expect(pass(scoreOutput({ tools: ["bash", "write"] }, obs), "tools")).toBe(false);
+  });
+
+  it("cost: at or under budget", () => {
+    expect(pass(scoreOutput({ max_cost_usd: 0.05 }, { output: "x", outcome: "ok", cost_usd: 0.02, tools: [] }), "cost")).toBe(true);
+    expect(pass(scoreOutput({ max_cost_usd: 0.05 }, { output: "x", outcome: "ok", cost_usd: 0.10, tools: [] }), "cost")).toBe(false);
+  });
+
+  it("only emits scores for declared expectations (plus completion)", () => {
+    const names = scoreOutput({ contains: ["a"] }, { output: "a", outcome: "ok", tools: [] }).map((s) => s.name);
+    expect(names).toEqual(["completed", "contains"]);
+  });
+});