diff --git a/.github/workflows/site-generation-evals.yml b/.github/workflows/site-generation-evals.yml new file mode 100644 index 0000000..3b66ac6 --- /dev/null +++ b/.github/workflows/site-generation-evals.yml @@ -0,0 +1,145 @@ +--- +name: site-generation-evals + +# Manual only: each run calls real agents and a judge model, so it costs +# tokens. Nothing here touches the `test` workflow. +on: + workflow_dispatch: + inputs: + scenarios: + description: "Scenario filter, e.g. 'hard-input/*' or 'table-stakes/bakery' (blank = all)" + required: false + default: "" + agents: + description: "Agent filter, e.g. 'claude-code' (blank = all in axis.config.ts)" + required: false + default: "" + baseline: + description: "Compare against evals/site-generation/baselines/.json and fail on regressions (blank = skip)" + required: false + default: "" + pairwise: + description: "Also run pairwise visual judging, claude-code vs codex" + type: boolean + default: false + +concurrency: + group: site-generation-evals-${{ github.ref }} + cancel-in-progress: false + +permissions: + contents: read + +jobs: + evals: + runs-on: ubuntu-latest + timeout-minutes: 120 + env: + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + CODEX_API_KEY: ${{ secrets.CODEX_API_KEY }} + EVALS: evals/site-generation + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + + - uses: actions/setup-node@v6 + with: + node-version: "24" + cache: npm + + - name: Install dependencies + run: npm ci --no-audit + + # dist/cli.js is the axis binary; the eval configs import from src/ via jiti. + - name: Build + run: npm run build + + - name: Install agent CLIs + run: npm install -g @anthropic-ai/claude-code @openai/codex + + # axis exits 1 if any job failed, but afterAll (metrics) has already run by + # then. Keep going so the summary and reports are still published; the + # last step fails the job. + - name: Run evals + id: run + continue-on-error: true + env: + SCENARIOS: ${{ inputs.scenarios }} + AGENTS: ${{ inputs.agents }} + run: | + args=(run -c "$EVALS/axis.config.ts") + if [ -n "$SCENARIOS" ]; then args+=(-s "$SCENARIOS"); fi + if [ -n "$AGENTS" ]; then args+=(-a "$AGENTS"); fi + node dist/cli.js "${args[@]}" + + - name: Metrics summary + if: always() + run: | + { + echo "## Site generation metrics" + echo '```' + npx jiti "$EVALS/lib/metrics.ts" latest + echo '```' + echo "By scenario:" + echo '```' + npx jiti "$EVALS/lib/metrics.ts" latest --by-scenario + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + + - name: Compare against baseline + if: inputs.baseline != '' + env: + BASELINE: ${{ inputs.baseline }} + run: | + if [ ! -f "$EVALS/baselines/$BASELINE.json" ]; then + echo "::error::No baseline at $EVALS/baselines/$BASELINE.json. Save and commit one first." + exit 1 + fi + set -o pipefail + { + echo "## Against baseline \`$BASELINE\`" + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + status=0 + npx jiti "$EVALS/lib/metrics.ts" latest --compare "$BASELINE" | tee -a "$GITHUB_STEP_SUMMARY" || status=$? + echo '```' >> "$GITHUB_STEP_SUMMARY" + exit $status + + - name: Pairwise visual judging + if: inputs.pairwise + env: + PAIRWISE_A: latest:claude-code + PAIRWISE_B: latest:codex + run: | + node dist/cli.js run -c "$EVALS/pairwise.config.ts" --no-score + { + echo "## Pairwise: A=claude-code vs B=codex" + echo '```' + npx jiti "$EVALS/lib/pairwise.ts" latest + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + + # Agents with a pinned model write files like 'claude-code|claude-opus-5-5.json', + # and upload-artifact rejects '|' in paths, so ship the tree as one tarball. + - name: Archive reports + if: always() + run: | + if [ -d "$EVALS/.axis/reports" ]; then + tar -czf "site-generation-reports-${GITHUB_RUN_ID}.tar.gz" -C "$EVALS/.axis" reports + fi + + - name: Upload reports + if: always() + uses: actions/upload-artifact@v4 + with: + name: site-generation-reports-${{ github.run_id }} + path: site-generation-reports-${{ github.run_id }}.tar.gz + if-no-files-found: warn + retention-days: 30 + + - name: Fail if any eval job failed + if: steps.run.outcome == 'failure' + run: | + echo "::error::One or more scenario/agent jobs failed. See the uploaded reports." + exit 1 diff --git a/evals/site-generation/README.md b/evals/site-generation/README.md new file mode 100644 index 0000000..609edbb --- /dev/null +++ b/evals/site-generation/README.md @@ -0,0 +1,122 @@ +# Site generation evals + +Reference-free, LLM-as-a-judge scenarios for site-building agents (Agent Runners). Everything here is built on hooks AXIS already has (inline scenarios, `teardown`, `artifacts`, `afterAll`, the `-c` flag), so the AXIS flow is unchanged: no edits to `src/`, the repo-root `scenarios/`, `npm test`, or CI. + +```bash +npm run build + +# Run the evals. Metrics are written to the report dir (metrics.md / metrics.json) when the run ends. +npx axis run -c evals/site-generation/axis.config.ts +npx axis run -c evals/site-generation/axis.config.ts -s 'hard-input/*' + +# Re-print metrics, or track them against a baseline +npx jiti evals/site-generation/lib/metrics.ts [report-id|latest] [--by-scenario] [--json] +npx jiti evals/site-generation/lib/metrics.ts --save-baseline main +npx jiti evals/site-generation/lib/metrics.ts --compare main [--threshold 0.05] # exits 1 on a regression + +# Pairwise visual judging between two treatments (see below) +PAIRWISE_A=latest:claude-code PAIRWISE_B=latest:codex \ + npx axis run -c evals/site-generation/pairwise.config.ts --no-score + +# Judge calibration against human labels (see below) +npx jiti evals/site-generation/lib/calibrate.ts template latest --sample 60 +npx jiti evals/site-generation/lib/calibrate.ts score evals/site-generation/golden/.json + +# This folder's own tests (not part of npm test) +npx vitest run --config evals/site-generation/vitest.config.ts +``` + +## Layout + +| Path | What's in it | +| ------------------------ | ----------------------------------------------------------------------- | +| `specs/**/*.json` | One scenario per file. The file path is the scenario key. | +| `rubric/core.json` | Checks every site gets: functional, JS errors, mobile, a11y, visual | +| `rubric/site-types.json` | Table-stakes checklists per kind of site (local business, event, …) | +| `rubric/audits.json` | Reusable precision audits, e.g. `facts` | +| `fixtures/` | Files copied into the workspace by a spec's `setup` (e.g. an upload) | +| `lib/compile.ts` | Turns specs + rubric into AXIS scenarios. You shouldn't need to edit it | +| `lib/checks.mjs` | Script checks, run in each workspace at teardown | +| `lib/metrics.ts` | Recall / precision / F1 / pass rate per vertical, plus baselines | +| `lib/pairwise.ts` | Builds pairwise scenarios and summarizes win rates | +| `lib/calibrate.ts` | Golden-label templates and judge-vs-human agreement | +| `baselines/`, `golden/` | Created on first use. Commit them to track metrics and labels over time | + +## How it hooks into AXIS + +``` +agent builds ./site/ → AXIS judge grades tagged checks → teardown: checks.mjs → artifacts captured → afterAll: metrics.ts --write +``` + +| Capability | AXIS hook used | +| ---------------------- | ----------------------------------------------------------------------------------------------------------------------------- | +| JSON specs | `scenarios` accepts inline scenarios from `axis.config.ts` | +| Recall/precision/pass | Tagged judge criteria; counts parsed from the report's per-check grades | +| Script checks | Scenario `teardown` writes `site-checks.json`, captured via `artifacts`; also summarized in the report through `$AXIS_OUTPUT` | +| Metrics after each run | `afterAll` with `$AXIS_REPORT_DIR` | +| Pairwise judging | A second config whose agent is the judge; `setup` copies the two saved sites | +| Baselines, calibration | Scripts that read finished reports | + +## Writing a spec + +```json +{ + "name": "Local business site from a complete brief", + "about": "Note for humans. Not sent to the agent or the judge.", + "site_type": "local-business", + "prompt": ["Lines are joined with spaces.", "So long prompts stay readable."], + "picker": [{ "question": "Pick a style", "answer": "Warm & organic" }], + "setup": [{ "action": "copy", "match": "fixtures/uploaded-logo/*", "destination": "uploads" }], + "rubric": { + "recall": { "content": ["The business name \"Crumb & Co.\", written exactly"] }, + "precision": ["facts"], + "pass": { "input-handling": ["The site has no blog section"] } + } +} +``` + +Everything except `name`, `prompt`, and `rubric` is optional. The compiler adds the site type's table stakes and the core checks, appends the picker selections plus an output contract (build static files in `./site/`) to the prompt, and adds the script checks to `teardown`. Unknown fields, site types, audits, or verticals fail loudly. + +## Metrics instead of weights + +There are no weights. The judge makes a yes/no call on every item and the metrics are computed from the counts: + +| Metric | Source | What's counted | Formula | +| ------------- | ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------- | +| **Recall** | judge, `recall` | PRESENT or MISSING for each item that should be on the site | present / listed | +| **Precision** | judge, `precision` | Every matching item on the site, labelled SUPPORTED, PLACEHOLDER, UNSUPPORTED | supported / (supported + unsupported) | +| **F1** | — | Balances precision and recall, for verticals that have both | 2PR / (P + R) | +| **Pass** | judge, `pass` | PASS or FAIL for each conformance check | passed / listed | +| **Check** | `checks.mjs` | Exact counts: refs that resolve, scripts that parse, pages with viewport/lang/one h1/landmarks, images with alt, declared color pairs meeting AA | passed / inspected | + +- Placeholders are left out of precision, so `[Your phone]` isn't penalised but a realistic invented phone number is. +- Precision audits have to write `supported=N placeholder=N unsupported=N` at the start of their rationale. If the judge skips it, the metric falls back to the 0–10 score and is marked `~`. +- Counts are pooled across runs and scenarios, so repeated runs tighten the numbers instead of averaging averages. +- `pass` and `check` overlap on purpose for mobile, accessibility, functional, and JS errors: where the judge and the script disagree, one of them is wrong. +- `checks.mjs` reads HTML/CSS without a browser. It counts what's declared (e.g. color pairs set in the same CSS rule), not what renders. +- The AXIS goal score still gets computed (it's now the share of checks met), but the numbers to read are the per-vertical ones. + +Verticals: `content`, `table-stakes`, `input-handling`, `imagery`, `functional`, `js-errors`, `mobile`, `accessibility`, `visual`. + +## Comparing treatments + +**Per-vertical metrics.** Add agent entries to `axis.config.ts`: the same adapter with two models (Model A vs Model B), or the same model with different flags (Config A vs Config B). `metrics.ts` prints one column per agent. The judge is pinned to one model so every treatment is graded the same way. + +**Pairwise visual judging.** `pairwise.config.ts` takes two treatments as `:`, from one report or two. For each brief both treatments built a site for, the judge (running as the agent, default `claude-opus-5-5`, override with `PAIRWISE_JUDGE_MODEL`) gets both sites as `left/` and `right/`. It picks a winner on typography, spacing, composition, color, overall, and fit-to-brief. Every brief is judged twice with the sides swapped: + +- win rate counts ties as half; +- `order-consistent` shows how often both orderings agreed. A low value means the judge is favouring a position, not a site. + +Results land in the pairwise report's `pairwise.md`; re-print with `npx jiti evals/site-generation/lib/pairwise.ts [report]`. Pairwise runs share `.axis/reports/` with normal runs, and `latest` picks the right kind for each script. + +**Baselines.** `--save-baseline ` snapshots per-vertical metrics into `baselines/.json`. `--compare ` prints deltas and exits 1 if any metric drops more than `--threshold` (default 0.05), so it can gate CI on e.g. accessibility without averaging it into content. + +## Calibrating the judge + +1. `calibrate.ts template [--sample N]` writes `golden/.json` with one label per check. It copies the sites into `golden//sites/` so the set outlives `.axis/`, and leaves out the judge's verdicts so labellers aren't anchored by them. +2. People fill in `human`: `true`/`false` for recall and pass checks, `{"supported": N, "unsupported": N}` for audits. +3. `calibrate.ts score ` reports, per vertical: + - the judge's accuracy, precision, recall, and Cohen's κ (agreement with humans corrected for chance) on yes/no checks; + - the mean gap between judge and human precision on audits. + +A vertical with low κ isn't ready to be trusted, whatever its score says. diff --git a/evals/site-generation/axis.config.ts b/evals/site-generation/axis.config.ts new file mode 100644 index 0000000..d70068c --- /dev/null +++ b/evals/site-generation/axis.config.ts @@ -0,0 +1,31 @@ +import type { AxisConfig } from "../../src/types/index.js"; +import { loadSpecs } from "./lib/compile.js"; + +/** + * Reference-free, LLM-as-a-judge evals for site generation (Agent Runners). + * Scenarios are authored as JSON in `specs/` and compiled by `lib/compile.ts`. + * Everything here rides on existing AXIS hooks (inline scenarios, teardown, + * artifacts, afterAll), so the AXIS flow itself is unchanged. See README.md. + * + * npx axis run -c evals/site-generation/axis.config.ts + * + * Pairwise comparisons across agents come from the agents list: add a second + * entry with a different model (Model A vs Model B) or adapter flags (Config A + * vs Config B). Pairwise visual judging lives in `pairwise.config.ts`. + */ +export default { + name: "Site generation", + scenarios: loadSpecs(), + agents: ["claude-code", "codex"], + judging: { + agents: [{ agent: "claude-code", model: "claude-opus-5-5" }], + }, + artifacts: ["site/**", "site-checks.json"], + settings: { + limits: { + scenario: { time_minutes: 15 }, + }, + }, + // Writes metrics.json / metrics.md into the report dir once the report is finalized. + afterAll: [{ action: "run_script", command: 'npx jiti lib/metrics.ts "$AXIS_REPORT_DIR" --write' }], +} satisfies AxisConfig; diff --git a/evals/site-generation/fixtures/uploaded-logo/logo.svg b/evals/site-generation/fixtures/uploaded-logo/logo.svg new file mode 100644 index 0000000..8344114 --- /dev/null +++ b/evals/site-generation/fixtures/uploaded-logo/logo.svg @@ -0,0 +1,8 @@ + + + + + + Fernhill + PLANT CO. + diff --git a/evals/site-generation/lib/calibrate.ts b/evals/site-generation/lib/calibrate.ts new file mode 100644 index 0000000..4a34b2d --- /dev/null +++ b/evals/site-generation/lib/calibrate.ts @@ -0,0 +1,200 @@ +import * as fs from "node:fs"; +import * as path from "node:path"; +import { fileURLToPath } from "node:url"; +import { ROOT, TAG_RE } from "./compile.js"; +import { flag, loadReport, renderTable, tallyCriterion } from "./metrics.js"; + +/** + * Judge calibration against human labels. + * + * template [--sample N] write golden/.json for people to fill in + * score compare the judge's calls to the human labels + * + * The template copies each labelled site into golden//sites/ so the + * golden set outlives `.axis/`, and leaves out the judge's verdicts so labellers + * aren't anchored by them. + */ + +export interface GoldenLabel { + scenario: string; + agent: string; + check: string; + /** Relative to the golden file. Open `index.html` to label. */ + site: string; + /** Recall/pass checks: true = met. Precision audits: counts of what the human found. */ + human: boolean | { supported: number | null; unsupported: number | null } | null; + note?: string; +} + +export interface GoldenFile { + report: string; + instructions: string; + labels: GoldenLabel[]; +} + +const INSTRUCTIONS = + "For each label, open the site and decide the check yourself before looking at any AXIS report. " + + "Recall/pass checks: set human to true (met) or false (not met). " + + 'Precision audits: set human to {"supported": N, "unsupported": N} using the audit\'s own definitions; placeholders are not counted. ' + + "Leave human as null to skip a label. Use note for anything ambiguous."; + +export function buildTemplate(reportRef: string, sample?: number, root = ROOT): GoldenFile { + const { manifest, dir } = loadReport(reportRef, root); + const goldenDir = path.join(root, "golden"); + const sitesDir = path.join(goldenDir, manifest.reportId, "sites"); + let labels: GoldenLabel[] = []; + + for (const result of manifest.results) { + const criteria = result.score?.goalAchievement?.criteria; + const site = path.join(dir, "scenarios", result.scenarioKey, result.agentName, "artifacts", "site"); + if (!criteria?.length || !fs.existsSync(site)) continue; + const dest = path.join(sitesDir, result.scenarioKey, result.agentName); + fs.mkdirSync(dest, { recursive: true }); + fs.cpSync(site, dest, { recursive: true }); + for (const c of criteria) { + if (!TAG_RE.test(c.check)) continue; + const isAudit = c.check.includes(":precision]"); + labels.push({ + scenario: result.scenarioKey, + agent: result.agentName, + check: c.check, + site: path.relative(goldenDir, dest), + human: isAudit ? { supported: null, unsupported: null } : null, + }); + } + } + if (sample !== undefined && sample < labels.length) labels = seededSample(labels, sample, manifest.reportId); + return { report: manifest.reportId, instructions: INSTRUCTIONS, labels }; +} + +export interface Agreement { + n: number; + /** Share of labels where judge and human agree. */ + accuracy: number; + /** Of the checks the judge called met, the share humans agree were met. */ + precision?: number; + /** Of the checks humans called met, the share the judge also called met. */ + recall?: number; + /** Agreement corrected for chance. ~0 = no better than chance, 1 = perfect. */ + kappa?: number; +} + +export interface Calibration { + report: string; + /** Recall/pass checks, per vertical plus `all`. */ + binary: Record; + /** Precision audits: mean absolute gap between judge and human precision, per vertical plus `all`. */ + audits: Record; + unmatched: number; +} + +export function agreement(pairs: { judge: boolean; human: boolean }[]): Agreement { + const n = pairs.length; + const tp = pairs.filter((p) => p.judge && p.human).length; + const fp = pairs.filter((p) => p.judge && !p.human).length; + const fn = pairs.filter((p) => !p.judge && p.human).length; + const tn = n - tp - fp - fn; + const po = (tp + tn) / n; + const pe = ((tp + fp) / n) * ((tp + fn) / n) + ((fn + tn) / n) * ((fp + tn) / n); + return { + n, + accuracy: po, + ...(tp + fp > 0 ? { precision: tp / (tp + fp) } : {}), + ...(tp + fn > 0 ? { recall: tp / (tp + fn) } : {}), + ...(pe < 1 ? { kappa: (po - pe) / (1 - pe) } : {}), + }; +} + +export function scoreGolden(golden: GoldenFile, root = ROOT): Calibration { + const { manifest } = loadReport(golden.report, root); + const binary = new Map(); + const audits = new Map(); + let unmatched = 0; + + for (const label of golden.labels) { + if (label.human === null) continue; + const grade = manifest.results + .find((r) => r.scenarioKey === label.scenario && r.agentName === label.agent) + ?.score?.goalAchievement?.criteria.find((c) => c.check === label.check); + const parsed = grade && tallyCriterion(grade); + if (!parsed) { + unmatched++; + continue; + } + for (const key of [parsed.vertical, "all"]) { + if (typeof label.human === "boolean") { + binary.set(key, [...(binary.get(key) ?? []), { judge: parsed.tally.hits === 1, human: label.human }]); + } else if (label.human.supported !== null && label.human.unsupported !== null) { + const humanTotal = label.human.supported + label.human.unsupported; + const humanP = humanTotal === 0 ? 1 : label.human.supported / humanTotal; + const judgeP = parsed.tally.total === 0 ? 1 : parsed.tally.hits / parsed.tally.total; + audits.set(key, [...(audits.get(key) ?? []), Math.abs(judgeP - humanP)]); + } + } + } + + return { + report: golden.report, + binary: Object.fromEntries([...binary].map(([k, pairs]) => [k, agreement(pairs)])), + audits: Object.fromEntries( + [...audits].map(([k, gaps]) => [k, { n: gaps.length, mae: gaps.reduce((a, b) => a + b, 0) / gaps.length }]), + ), + unmatched, + }; +} + +export function formatCalibration(c: Calibration): string { + const f = (n: number | undefined) => (n === undefined ? "" : n.toFixed(2)); + const sortAllLast = (keys: string[]) => + keys.sort((a, b) => (a === "all" ? 1 : b === "all" ? -1 : a.localeCompare(b))); + const binaryRows = sortAllLast(Object.keys(c.binary)).map((k) => { + const a = c.binary[k]; + return [k, String(a.n), f(a.accuracy), f(a.precision), f(a.recall), f(a.kappa)]; + }); + const auditRows = sortAllLast(Object.keys(c.audits)).map((k) => [k, String(c.audits[k].n), f(c.audits[k].mae)]); + return ( + `Judge vs human labels, report ${c.report}\n\n` + + renderTable(["vertical", "n", "accuracy", "precision", "recall", "kappa"], binaryRows) + + (auditRows.length ? "\nPrecision audits\n\n" + renderTable(["vertical", "n", "mean abs gap"], auditRows) : "") + + (c.unmatched ? `\n${c.unmatched} label(s) didn't match a graded check in the report\n` : "") + ); +} + +/** Deterministic sample so re-running `template` gives labellers the same set. */ +function seededSample(items: T[], n: number, seed: string): T[] { + let h = [...seed].reduce((acc, ch) => (acc * 31 + ch.charCodeAt(0)) >>> 0, 7); + const rand = () => (h = (h * 1103515245 + 12345) >>> 0) / 2 ** 32; + const copy = [...items]; + for (let i = copy.length - 1; i > 0; i--) { + const j = Math.floor(rand() * (i + 1)); + [copy[i], copy[j]] = [copy[j], copy[i]]; + } + return copy.slice(0, n); +} + +function main(argv: string[]): void { + const [command, target] = argv.filter((a, i) => !a.startsWith("--") && argv[i - 1] !== "--sample"); + if (command === "template") { + const sample = flag(argv, "--sample"); + const golden = buildTemplate(target ?? "latest", sample === undefined ? undefined : Number(sample)); + const out = path.join(ROOT, "golden", `${golden.report}.json`); + if (fs.existsSync(out)) throw new Error(`${out} already exists; refusing to overwrite human labels.`); + fs.writeFileSync(out, JSON.stringify(golden, null, 2) + "\n"); + process.stdout.write(`Wrote ${golden.labels.length} labels to ${path.relative(process.cwd(), out)}\n`); + } else if (command === "score" && target) { + const golden = JSON.parse(fs.readFileSync(target, "utf-8")) as GoldenFile; + const calibration = scoreGolden(golden); + process.stdout.write( + argv.includes("--json") ? JSON.stringify(calibration, null, 2) + "\n" : formatCalibration(calibration), + ); + } else { + process.stderr.write( + "Usage: calibrate.ts template [--sample N]\n calibrate.ts score \n", + ); + process.exitCode = 1; + } +} + +if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + main(process.argv.slice(2)); +} diff --git a/evals/site-generation/lib/checks.mjs b/evals/site-generation/lib/checks.mjs new file mode 100644 index 0000000..e937250 --- /dev/null +++ b/evals/site-generation/lib/checks.mjs @@ -0,0 +1,190 @@ +#!/usr/bin/env node +// Script checks for a generated site, run from each scenario's teardown in +// the agent workspace. No dependencies and no browser: regex-level HTML/CSS +// inspection, so treat results as exact counts of what's *declared*, not a +// rendered-page audit. Writes `site-checks.json` (captured as an artifact) and +// a short summary to $AXIS_OUTPUT so it shows in the AXIS report. +import * as fs from "node:fs"; +import * as path from "node:path"; +import * as vm from "node:vm"; + +const SITE = path.resolve(process.argv[2] ?? "site"); +const OUT = path.resolve(process.argv[3] ?? "site-checks.json"); + +const checks = []; +function record(vertical, id, hits, total, detail = []) { + checks.push({ vertical, id, hits, total, detail: detail.slice(0, 20) }); +} + +const files = fs.existsSync(SITE) ? walk(SITE) : []; +const pages = files.filter((f) => f.endsWith(".html")); +const html = new Map(pages.map((f) => [f, fs.readFileSync(f, "utf-8")])); +const css = files.filter((f) => f.endsWith(".css")).map((f) => fs.readFileSync(f, "utf-8")); +for (const source of html.values()) css.push(...matchAll(source, /]*>([\s\S]*?)<\/style>/gi).map((m) => m[1])); + +record("functional", "index-exists", fs.existsSync(path.join(SITE, "index.html")) ? 1 : 0, 1); + +// Local references (href/src) resolve to files; in-page anchors resolve to ids. +{ + let ok = 0; + let total = 0; + const broken = []; + for (const [file, source] of html) { + const ids = new Set(matchAll(source, /\sid=["']([^"']+)["']/gi).map((m) => m[1])); + for (const [, ref] of matchAll(source, /\s(?:href|src)=["']([^"']*)["']/gi)) { + if (/^(https?:|mailto:|tel:|data:|javascript:|\/\/)/i.test(ref) || ref === "") continue; + total++; + const [target, hash] = ref.split("#"); + const exists = target === "" ? hash === undefined || hash === "" || ids.has(hash) : resolveLocal(file, target); + if (exists) ok++; + else broken.push(`${rel(file)}: ${ref}`); + } + } + record("functional", "local-refs-resolve", ok, total, broken); +} + +// Scripts parse. Classic scripts are compiled (not run) with vm.Script; module +// scripts can't be, so they're counted as unknown rather than failures. +{ + let ok = 0; + let total = 0; + const errors = []; + const sources = []; + for (const [file, source] of html) { + for (const [, attrs, body] of matchAll(source, /]*)>([\s\S]*?)<\/script>/gi)) { + if (/type=["']module["']/i.test(attrs) || /\ssrc=/i.test(attrs)) continue; + if (/type=["'](?!text\/javascript|application\/javascript)[^"']+["']/i.test(attrs)) continue; + sources.push([`${rel(file)} inline`, body]); + } + } + for (const f of files.filter((f) => f.endsWith(".js"))) sources.push([rel(f), fs.readFileSync(f, "utf-8")]); + for (const [label, code] of sources) { + if (/^\s*(import|export)\s/m.test(code)) continue; + total++; + try { + new vm.Script(code, { filename: label }); + ok++; + } catch (err) { + errors.push(`${label}: ${err.message}`); + } + } + record("js-errors", "scripts-parse", ok, total, errors); +} + +perPage("mobile", "viewport-meta", (s) => /]+name=["']viewport["'][^>]*width=device-width/i.test(s)); +record( + "mobile", + "has-media-queries-or-fluid-layout", + css.some((c) => /@media|display:\s*(flex|grid)|clamp\(|minmax\(/i.test(c)) ? 1 : 0, + 1, +); +perPage("accessibility", "html-lang", (s) => /]*\slang=["'][^"']+["']/i.test(s)); +perPage("accessibility", "single-h1", (s) => matchAll(s, /]/gi).length === 1); +perPage("accessibility", "has-landmarks", (s) => + ["header", "nav", "main", "footer"].every((t) => new RegExp(`<${t}[\\s>]`, "i").test(s)), +); + +// Every has an alt attribute (empty alt is valid for decorative images). +{ + let ok = 0; + let total = 0; + const missing = []; + for (const [file, source] of html) { + for (const [tag] of matchAll(source, /]*>/gi)) { + total++; + if (/\salt=["'][^"']*["']/i.test(tag)) ok++; + else missing.push(`${rel(file)}: ${tag.slice(0, 80)}`); + } + } + record("accessibility", "img-alt", ok, total, missing); +} + +// Contrast for CSS rules that declare both a text color and a background +// color as hex/rgb literals. Custom properties and inherited colors aren't +// resolved, so this checks declared pairs only. +{ + let ok = 0; + let total = 0; + const failing = []; + const vars = new Map(); + for (const sheet of css) { + for (const [, name, value] of matchAll(sheet, /(--[\w-]+)\s*:\s*([^;}]+)/g)) vars.set(name, value.trim()); + } + const resolve = (value) => value.replace(/var\((--[\w-]+)[^)]*\)/g, (_, n) => vars.get(n) ?? ""); + for (const sheet of css) { + for (const [, selector, body] of matchAll(sheet, /([^{}]+)\{([^{}]*)\}/g)) { + const fg = /(?:^|;|\s)color\s*:\s*([^;]+)/i.exec(body)?.[1]; + const bg = /background(?:-color)?\s*:\s*([^;]+)/i.exec(body)?.[1]; + const a = fg && parseColor(resolve(fg)); + const b = bg && parseColor(resolve(bg)); + if (!a || !b) continue; + total++; + const ratio = contrast(a, b); + if (ratio >= 4.5) ok++; + else failing.push(`${selector.trim().slice(0, 60)}: ${ratio.toFixed(2)}:1`); + } + } + record("accessibility", "declared-contrast-aa", ok, total, failing); +} + +fs.writeFileSync(OUT, JSON.stringify({ site: path.basename(SITE), pages: pages.length, checks }, null, 2)); +if (process.env.AXIS_OUTPUT) { + const lines = checks.map((c) => `| ${c.vertical} | ${c.id} | ${c.hits}/${c.total} |`); + fs.appendFileSync( + process.env.AXIS_OUTPUT, + `### Site checks\n\n| Vertical | Check | Passed |\n| --- | --- | --- |\n${lines.join("\n")}\n`, + ); +} + +function perPage(vertical, id, test) { + const failing = [...html].filter(([, s]) => !test(s)).map(([f]) => rel(f)); + record(vertical, id, html.size - failing.length, html.size, failing); +} + +function resolveLocal(fromFile, target) { + const clean = decodeURIComponent(target.split("?")[0]); + const abs = clean.startsWith("/") ? path.join(SITE, clean) : path.resolve(path.dirname(fromFile), clean); + return fs.existsSync(abs) && (fs.statSync(abs).isFile() || fs.existsSync(path.join(abs, "index.html"))); +} + +function parseColor(value) { + const v = value.trim().toLowerCase(); + let m = /#([0-9a-f]{3,8})\b/.exec(v); + if (m) { + let hex = m[1]; + if (hex.length === 3 || hex.length === 4) hex = [...hex.slice(0, 3)].map((c) => c + c).join(""); + return [0, 2, 4].map((i) => parseInt(hex.slice(i, i + 2), 16)); + } + m = /rgba?\(\s*(\d+)[\s,]+(\d+)[\s,]+(\d+)/.exec(v); + if (m) return [m[1], m[2], m[3]].map(Number); + if (/\bwhite\b/.test(v)) return [255, 255, 255]; + if (/\bblack\b/.test(v)) return [0, 0, 0]; + return null; +} + +function contrast(a, b) { + const lum = (rgb) => { + const [r, g, bl] = rgb.map((c) => { + const s = c / 255; + return s <= 0.03928 ? s / 12.92 : ((s + 0.055) / 1.055) ** 2.4; + }); + return 0.2126 * r + 0.7152 * g + 0.0722 * bl; + }; + const [hi, lo] = [lum(a), lum(b)].sort((x, y) => y - x); + return (hi + 0.05) / (lo + 0.05); +} + +function matchAll(s, re) { + return [...s.matchAll(re)]; +} + +function rel(f) { + return path.relative(SITE, f) || path.basename(f); +} + +function walk(dir) { + return fs.readdirSync(dir, { withFileTypes: true }).flatMap((e) => { + const full = path.join(dir, e.name); + return e.isDirectory() ? (e.name === "node_modules" ? [] : walk(full)) : [full]; + }); +} diff --git a/evals/site-generation/lib/compile.ts b/evals/site-generation/lib/compile.ts new file mode 100644 index 0000000..acddfab --- /dev/null +++ b/evals/site-generation/lib/compile.ts @@ -0,0 +1,221 @@ +import * as fs from "node:fs"; +import * as path from "node:path"; +import { fileURLToPath } from "node:url"; +import type { InlineScenario, JudgeCriterion, LifecycleAction } from "../../../src/types/index.js"; + +export const VERTICALS = [ + "content", + "table-stakes", + "input-handling", + "imagery", + "functional", + "js-errors", + "mobile", + "accessibility", + "visual", +] as const; +export type Vertical = (typeof VERTICALS)[number]; +export type Metric = "recall" | "precision" | "pass"; + +/** A precision audit: the judge lists every matching item and labels it supported / placeholder / unsupported. */ +export interface Audit { + vertical: Vertical; + items: string; + supported: string; + unsupported: string; +} + +export interface Rubric { + /** Items that must be present. Recall = present / listed. */ + recall?: Partial>; + /** Audit ids from `rubric/audits.json`, or inline audits. */ + precision?: (string | Audit)[]; + /** PASS/FAIL checks. Pass rate = passed / listed. */ + pass?: Partial>; +} + +export interface Spec { + name: string; + /** Note for humans reading the spec; not sent to the agent or judge. */ + about?: string; + /** Picks the table-stakes checklist from `rubric/site-types.json`. */ + site_type?: string; + /** Lines are joined with spaces. */ + prompt: string | string[]; + /** Simulated option-picker selections, appended to the prompt. */ + picker?: { question: string; answer: string }[]; + setup?: LifecycleAction[]; + teardown?: LifecycleAction[]; + rubric: Rubric; +} + +interface Library { + core: Rubric; + siteTypes: Record; + audits: Record; +} + +export const ROOT = fileURLToPath(new URL("..", import.meta.url)); + +/** + * Every check is tagged `[vertical:metric]` so `lib/metrics.ts` can pull the + * grades back out of a report and compute recall, precision, and pass rate. + */ +export const TAG_RE = /^\[([a-z-]+):(recall|precision|pass)\] /; + +export const OUTPUT_CONTRACT = + "\n\n---\n" + + "Build the site as static files in `./site/`, with `site/index.html` as the entry point. " + + "Plain HTML, CSS, and JavaScript only, with no build step, so the site opens directly in a browser. " + + "Don't ask me follow-up questions. Make reasonable decisions and finish the site."; + +/** Script checks (links, script parsing, viewport, lang, h1, alt, declared contrast). See lib/checks.mjs. */ +export const SITE_CHECKS: LifecycleAction = { + action: "run_script", + command: 'node "$AXIS_CONFIG_DIR/lib/checks.mjs" site site-checks.json || true', +}; + +const SPEC_KEYS = new Set(["name", "about", "site_type", "prompt", "picker", "setup", "teardown", "rubric"]); +const RUBRIC_KEYS = new Set(["$comment", "recall", "precision", "pass"]); + +export function recallCheck(vertical: Vertical, item: string): string { + return ( + `[${vertical}:recall] PRESENT (10) or MISSING (0), no partial credit. ` + + `Placeholder values count as present unless this asks for a specific value. The site includes: ${item}` + ); +} + +export function passCheck(vertical: Vertical, check: string): string { + return `[${vertical}:pass] PASS (10) or FAIL (0), no partial credit: ${check}`; +} + +export function precisionCheck(audit: Audit): string { + return ( + `[${audit.vertical}:precision] AUDIT ${audit.items}. List every one on the site and label it ` + + `SUPPORTED (${audit.supported}), PLACEHOLDER (clearly marked for the owner to fill in, e.g. "[Your phone]"), ` + + `or UNSUPPORTED (${audit.unsupported}). ` + + `Start the rationale with exactly "supported=N placeholder=N unsupported=N", then name each unsupported item. ` + + `Score = round(10 × supported ÷ (supported + unsupported)), or 10 if both are 0.` + ); +} + +export function buildPrompt(spec: Spec): string { + let prompt = Array.isArray(spec.prompt) ? spec.prompt.join(" ") : spec.prompt; + if (spec.picker?.length) { + prompt += "\n\nOptions selected:\n" + spec.picker.map((p) => `- ${p.question}: ${p.answer}`).join("\n"); + } + return prompt + OUTPUT_CONTRACT; +} + +/** Scenario-specific checks first, then the site type's table stakes, then the core checks every site gets. */ +export function buildJudge(spec: Spec, lib: Library, where: string): JudgeCriterion[] { + const rubrics: Rubric[] = [spec.rubric]; + if (spec.site_type !== undefined) { + const siteType = lib.siteTypes[spec.site_type]; + if (!siteType) { + throw new Error(`${where}: unknown site_type "${spec.site_type}" (see rubric/site-types.json)`); + } + rubrics.push(siteType); + } + rubrics.push(lib.core); + + const checks: string[] = []; + for (const rubric of rubrics) { + for (const [vertical, items] of verticalEntries(rubric.recall, where)) { + checks.push(...items.map((item) => recallCheck(vertical, item))); + } + for (const entry of rubric.precision ?? []) { + const audit = typeof entry === "string" ? lib.audits[entry] : entry; + if (!audit) throw new Error(`${where}: unknown precision audit "${entry}" (see rubric/audits.json)`); + assertVertical(audit.vertical, where); + checks.push(precisionCheck(audit)); + } + for (const [vertical, items] of verticalEntries(rubric.pass, where)) { + checks.push(...items.map((check) => passCheck(vertical, check))); + } + } + // No weights: each item counts once. The AXIS goal score becomes the share of checks met; + // the per-vertical metrics come from lib/metrics.ts. + return checks.map((check) => ({ check })); +} + +export function compileSpec(spec: Spec, key: string, lib: Library): InlineScenario { + const where = `specs/${key}.json`; + for (const k of Object.keys(spec)) { + if (!SPEC_KEYS.has(k)) throw new Error(`${where}: unknown field "${k}"`); + } + if (!spec.name || !spec.prompt || !spec.rubric) { + throw new Error(`${where}: "name", "prompt", and "rubric" are required`); + } + for (const k of Object.keys(spec.rubric)) { + if (!RUBRIC_KEYS.has(k)) throw new Error(`${where}: unknown rubric field "${k}"`); + } + + return { + key, + name: spec.name, + prompt: buildPrompt(spec), + judge: buildJudge(spec, lib, where), + ...(spec.setup ? { setup: spec.setup } : {}), + // Teardown runs after the judge and before artifact capture, so the script + // checks' `site-checks.json` lands in the report without touching scoring. + teardown: [SITE_CHECKS, ...(spec.teardown ?? [])], + }; +} + +export function loadLibrary(root = ROOT): Library { + return { + core: readJson(path.join(root, "rubric", "core.json")), + siteTypes: withoutComment(readJson(path.join(root, "rubric", "site-types.json"))), + audits: withoutComment(readJson(path.join(root, "rubric", "audits.json"))), + }; +} + +/** Compile every `specs/**\/*.json` into an AXIS inline scenario keyed by its path (`hard-input/vague`). */ +export function loadSpecs(root = ROOT): InlineScenario[] { + const lib = loadLibrary(root); + const specsDir = path.join(root, "specs"); + return walk(specsDir) + .filter((file) => file.endsWith(".json")) + .sort() + .map((file) => { + const key = path + .relative(specsDir, file) + .replace(/\.json$/, "") + .split(path.sep) + .join("/"); + return compileSpec(readJson(file), key, lib); + }); +} + +function verticalEntries(section: Partial> | undefined, where: string) { + const entries = Object.entries(section ?? {}) as [Vertical, string[]][]; + for (const [vertical] of entries) assertVertical(vertical, where); + return entries; +} + +function assertVertical(vertical: string, where: string): asserts vertical is Vertical { + if (!(VERTICALS as readonly string[]).includes(vertical)) { + throw new Error(`${where}: unknown vertical "${vertical}" (expected one of ${VERTICALS.join(", ")})`); + } +} + +function readJson(file: string): T { + try { + return JSON.parse(fs.readFileSync(file, "utf-8")) as T; + } catch (err) { + throw new Error(`Could not read ${file}: ${(err as Error).message}`); + } +} + +function withoutComment(record: Record): Record { + const { $comment: _comment, ...rest } = record; + return rest; +} + +function walk(dir: string): string[] { + return fs.readdirSync(dir, { withFileTypes: true }).flatMap((entry) => { + const full = path.join(dir, entry.name); + return entry.isDirectory() ? walk(full) : [full]; + }); +} diff --git a/evals/site-generation/lib/metrics.ts b/evals/site-generation/lib/metrics.ts new file mode 100644 index 0000000..6e3c6e4 --- /dev/null +++ b/evals/site-generation/lib/metrics.ts @@ -0,0 +1,287 @@ +import * as fs from "node:fs"; +import * as path from "node:path"; +import { fileURLToPath } from "node:url"; +import type { CriterionGrade, ReportManifest } from "../../../src/types/index.js"; +import { ROOT, TAG_RE, type Metric } from "./compile.js"; + +/** Recall and pass checks count as met at score ≥ 5, so a stray 7 from the judge still reads as a yes. */ +const MET_THRESHOLD = 5; +const COUNTS_RE = /supported\s*=\s*(\d+)\s*,?\s*placeholder\s*=\s*(\d+)\s*,?\s*unsupported\s*=\s*(\d+)/i; + +/** `check` is the script-check pass rate from lib/checks.mjs; the rest come from the LLM judge. */ +export type MetricKind = Metric | "check"; +const ROW_ORDER = ["recall", "precision", "f1", "pass", "check"] as const; + +export interface Tally { + hits: number; + total: number; + /** Precision audits whose rationale lacked parseable counts; hits/total fell back to score/10. */ + estimated: number; +} + +export type VerticalMetrics = Partial>; + +export interface Summary { + reportId: string; + by: "agent" | "scenario"; + /** Group label (agent name or scenario key) → vertical → metrics. */ + groups: Record>; + /** Runs with no goal-achievement grades (agent or judge failed), per group. */ + unscored: Record; +} + +export interface LoadedReport { + manifest: ReportManifest; + dir: string; +} + +interface SiteChecks { + checks: { vertical: string; id: string; hits: number; total: number }[]; +} + +/** Turn one graded criterion into counts. Untagged criteria (not from lib/compile.ts) return null. */ +export function tallyCriterion(grade: CriterionGrade): { vertical: string; metric: Metric; tally: Tally } | null { + const match = TAG_RE.exec(grade.check); + if (!match) return null; + const [, vertical, metric] = match as unknown as [string, string, Metric]; + + if (metric !== "precision") { + return { vertical, metric, tally: { hits: grade.score >= MET_THRESHOLD ? 1 : 0, total: 1, estimated: 0 } }; + } + const counts = COUNTS_RE.exec(grade.rationale); + if (!counts) { + return { vertical, metric, tally: { hits: grade.score / 10, total: 1, estimated: 1 } }; + } + const supported = Number(counts[1]); + const unsupported = Number(counts[3]); + return { vertical, metric, tally: { hits: supported, total: supported + unsupported, estimated: 0 } }; +} + +/** + * Pool counts per group × vertical × metric (micro-average), so running a + * scenario several times, or across several scenarios, just adds to the + * tallies. With `reportDir`, script checks captured as `site-checks.json` + * artifacts are pooled in as the `check` metric. + */ +export function summarize(manifest: ReportManifest, by: "agent" | "scenario", reportDir?: string): Summary { + const summary: Summary = { reportId: manifest.reportId, by, groups: {}, unscored: {} }; + const add = (group: string, vertical: string, metric: MetricKind, t: Tally) => { + const metrics = (summary.groups[group][vertical] ??= {}); + const tally = (metrics[metric] ??= { hits: 0, total: 0, estimated: 0 }); + tally.hits += t.hits; + tally.total += t.total; + tally.estimated += t.estimated; + }; + + for (const result of manifest.results) { + const group = by === "agent" ? result.agentName : result.scenarioKey; + summary.groups[group] ??= {}; + + const criteria = result.score?.goalAchievement?.criteria; + if (criteria?.length) { + for (const grade of criteria) { + const parsed = tallyCriterion(grade); + if (parsed) add(group, parsed.vertical, parsed.metric, parsed.tally); + } + } else { + summary.unscored[group] = (summary.unscored[group] ?? 0) + 1; + } + + const checks = reportDir && readSiteChecks(reportDir, result.scenarioKey, result.agentName); + for (const c of checks ? checks.checks : []) { + // A check with nothing to inspect (no images, no scripts) isn't evidence either way. + if (c.total > 0) add(group, c.vertical, "check", { hits: c.hits, total: c.total, estimated: 0 }); + } + } + return summary; +} + +export function readSiteChecks(reportDir: string, scenarioKey: string, agentName: string): SiteChecks | undefined { + const file = path.join(reportDir, "scenarios", scenarioKey, agentName, "artifacts", "site-checks.json"); + return fs.existsSync(file) ? (JSON.parse(fs.readFileSync(file, "utf-8")) as SiteChecks) : undefined; +} + +export function rate(tally: Tally | undefined): number | undefined { + if (!tally) return undefined; + // An audit that found nothing to label has nothing unsupported: precision 1. + return tally.total === 0 ? 1 : tally.hits / tally.total; +} + +export function f1(metrics: VerticalMetrics): number | undefined { + const p = rate(metrics.precision); + const r = rate(metrics.recall); + if (p === undefined || r === undefined) return undefined; + return p + r === 0 ? 0 : (2 * p * r) / (p + r); +} + +export function value(metrics: VerticalMetrics | undefined, kind: (typeof ROW_ORDER)[number]): number | undefined { + if (!metrics) return undefined; + return kind === "f1" ? f1(metrics) : rate(metrics[kind]); +} + +export function formatTable(summary: Summary): string { + const groups = Object.keys(summary.groups).sort(); + const verticals = [...new Set(groups.flatMap((g) => Object.keys(summary.groups[g])))].sort(); + const body: string[][] = []; + + for (const vertical of verticals) { + for (const kind of ROW_ORDER) { + const cells = groups.map((g) => { + const m = summary.groups[g][vertical]; + if (kind === "f1") return fmt(value(m, kind)); + const tally = m?.[kind]; + if (!tally) return ""; + const approx = tally.estimated > 0 ? "~" : ""; + return `${approx}${fmt(rate(tally))} (${round(tally.hits)}/${tally.total})`; + }); + if (cells.some((c) => c !== "")) body.push([`${vertical} ${kind}`, ...cells]); + } + } + body.push(["unscored runs", ...groups.map((g) => String(summary.unscored[g] ?? 0))]); + + return ( + renderTable(["", ...groups], body) + + "\npass = LLM judge PASS/FAIL · check = script checks (lib/checks.mjs)" + + "\n~ = includes precision audits without parseable counts (estimated from the 0–10 score)\n" + ); +} + +export interface Regression { + group: string; + vertical: string; + kind: string; + baseline: number; + current: number; +} + +/** Per-vertical deltas against a saved baseline. A drop larger than `threshold` is a regression. */ +export function compareSummaries(current: Summary, baseline: Summary, threshold: number) { + const rows: string[][] = []; + const regressions: Regression[] = []; + for (const group of Object.keys(current.groups).sort()) { + const verticals = Object.keys(current.groups[group]).sort(); + for (const vertical of verticals) { + for (const kind of ROW_ORDER) { + const now = value(current.groups[group][vertical], kind); + const then = value(baseline.groups[group]?.[vertical], kind); + if (now === undefined || then === undefined) continue; + const delta = now - then; + const flag = delta < -threshold ? "REGRESSED" : delta > threshold ? "improved" : ""; + if (flag === "REGRESSED") regressions.push({ group, vertical, kind, baseline: then, current: now }); + rows.push([group, `${vertical} ${kind}`, fmt(then), fmt(now), signed(delta), flag]); + } + } + } + return { table: renderTable(["group", "metric", "baseline", "current", "delta", ""], rows), regressions }; +} + +/** Pairwise runs (pairwise.config.ts) share the reports dir; `latest` picks the newest report of the requested kind. */ +export function isPairwiseReport(manifest: ReportManifest): boolean { + return manifest.results.length > 0 && manifest.results.every((r) => r.scenarioKey.startsWith("pairwise/")); +} + +/** Accepts a report id, `latest`, or a path to a report directory (what afterAll passes as $AXIS_REPORT_DIR). */ +export function loadReport(ref: string, root = ROOT, kind: "eval" | "pairwise" = "eval"): LoadedReport { + let dir: string; + if (fs.existsSync(path.join(ref, "report.json"))) { + dir = path.resolve(ref); + } else { + const reportsDir = path.join(root, ".axis", "reports"); + if (ref !== "latest") { + dir = path.join(reportsDir, ref); + } else { + const ids = fs.existsSync(reportsDir) ? fs.readdirSync(reportsDir).sort().reverse() : []; + const match = ids.find((id) => { + const file = path.join(reportsDir, id, "report.json"); + return fs.existsSync(file) && isPairwiseReport(readManifest(file)) === (kind === "pairwise"); + }); + if (!match) throw new Error(`No ${kind} reports in ${reportsDir}. Run the evals first.`); + dir = path.join(reportsDir, match); + } + } + return { manifest: readManifest(path.join(dir, "report.json")), dir }; +} + +function readManifest(file: string): ReportManifest { + return JSON.parse(fs.readFileSync(file, "utf-8")) as ReportManifest; +} + +export function renderTable(header: string[], body: string[][]): string { + const table = [header, ...body]; + const widths = header.map((_, i) => Math.max(...table.map((r) => (r[i] ?? "").length))); + const lines = table.map((r) => + r + .map((c, i) => (c ?? "").padEnd(widths[i])) + .join(" ") + .trimEnd(), + ); + lines.splice(1, 0, widths.map((w) => "-".repeat(w)).join(" ")); + return lines.join("\n") + "\n"; +} + +function fmt(n: number | undefined): string { + return n === undefined ? "" : n.toFixed(2); +} + +function signed(n: number): string { + return (n >= 0 ? "+" : "") + n.toFixed(2); +} + +function round(n: number): number { + return Math.round(n * 100) / 100; +} + +export function flag(argv: string[], name: string): string | undefined { + const i = argv.indexOf(name); + return i >= 0 ? argv[i + 1] : undefined; +} + +function main(argv: string[]): number { + const valueFlags = ["--save-baseline", "--compare", "--threshold"]; + const positional = argv.filter((a, i) => !a.startsWith("--") && !valueFlags.includes(argv[i - 1])); + const by = argv.includes("--by-scenario") ? "scenario" : "agent"; + const { manifest, dir } = loadReport(positional[0] ?? "latest"); + const summary = summarize(manifest, by, dir); + const table = formatTable(summary); + + if (argv.includes("--json")) { + process.stdout.write(JSON.stringify(summary, null, 2) + "\n"); + } else { + process.stdout.write(`Report ${manifest.reportId}, by ${by}\n\n${table}`); + } + + // --write: drop metrics next to the report (used by afterAll in axis.config.ts). + if (argv.includes("--write")) { + fs.writeFileSync(path.join(dir, "metrics.json"), JSON.stringify(summary, null, 2)); + fs.writeFileSync( + path.join(dir, "metrics.md"), + `# Metrics: ${manifest.reportId} (by ${by})\n\n\`\`\`\n${table}\`\`\`\n`, + ); + } + + const baselinesDir = path.join(ROOT, "baselines"); + const save = flag(argv, "--save-baseline"); + if (save) { + fs.mkdirSync(baselinesDir, { recursive: true }); + fs.writeFileSync(path.join(baselinesDir, `${save}.json`), JSON.stringify(summary, null, 2) + "\n"); + process.stdout.write(`\nSaved baseline "${save}" from report ${manifest.reportId}\n`); + } + + const compare = flag(argv, "--compare"); + if (compare) { + const baseline = JSON.parse(fs.readFileSync(path.join(baselinesDir, `${compare}.json`), "utf-8")) as Summary; + if (baseline.by !== by) throw new Error(`Baseline "${compare}" is grouped by ${baseline.by}; pass matching flags.`); + const threshold = Number(flag(argv, "--threshold") ?? 0.05); + const { table: diff, regressions } = compareSummaries(summary, baseline, threshold); + process.stdout.write( + `\nAgainst baseline "${compare}" (report ${baseline.reportId}), threshold ${threshold}\n\n${diff}`, + ); + process.stdout.write(`\n${regressions.length} regression(s)\n`); + return regressions.length > 0 ? 1 : 0; + } + return 0; +} + +if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + process.exitCode = main(process.argv.slice(2)); +} diff --git a/evals/site-generation/lib/pairwise.ts b/evals/site-generation/lib/pairwise.ts new file mode 100644 index 0000000..8af557e --- /dev/null +++ b/evals/site-generation/lib/pairwise.ts @@ -0,0 +1,204 @@ +import * as fs from "node:fs"; +import * as path from "node:path"; +import { fileURLToPath } from "node:url"; +import type { InlineScenario, ReportManifest } from "../../../src/types/index.js"; +import { ROOT } from "./compile.js"; +import { loadReport, renderTable } from "./metrics.js"; + +/** + * Pairwise visual judging, built from two finished reports. The judge runs as + * the *agent* of an ordinary AXIS scenario: setup copies both saved sites into + * `left/` and `right/`, and the judge writes `verdict.json`, captured as an + * artifact. Each pair runs in both orders (`@a-left`, `@b-left`) to cancel out + * position bias. Nothing in the AXIS flow changes; see pairwise.config.ts. + */ + +export const DIMENSIONS = ["typography", "spacing", "composition", "color", "overall", "fit-to-brief"] as const; +export type Dimension = (typeof DIMENSIONS)[number]; +type Side = "left" | "right" | "tie"; + +export interface Treatment { + /** Report id, `latest`, or report directory. */ + report: string; + agent: string; +} + +/** Parse `reportRef:agentName`. The agent name may itself contain colons, so split on the first one only. */ +export function parseTreatment(value: string | undefined, envName: string): Treatment { + const i = value?.indexOf(":") ?? -1; + if (!value || i <= 0) { + throw new Error(`Set ${envName}=:, e.g. ${envName}=latest:claude-code`); + } + return { report: value.slice(0, i), agent: value.slice(i + 1) }; +} + +function siteDir(reportDir: string, scenarioKey: string, agent: string): string { + return path.join(reportDir, "scenarios", scenarioKey, agent, "artifacts", "site"); +} + +/** The brief without the output contract lib/compile.ts appends. */ +export function briefOf(prompt: string | undefined): string { + return (prompt ?? "").split("\n\n---\n")[0].trim(); +} + +export function pairwisePrompt(brief: string): string { + const dims = DIMENSIONS.map((d) => `"${d}"`).join(", "); + return ( + "You are judging two websites built from the same brief. They are in `left/` and `right/` " + + "(each has an `index.html`). Do not modify them.\n\n" + + `THE BRIEF:\n${brief}\n\n` + + 'Compare them on each dimension below and pick the better one, or "tie" only if they are genuinely equal:\n' + + "- typography: type scale, readability, font pairing\n" + + "- spacing: consistent rhythm, breathing room, alignment\n" + + "- composition: visual hierarchy, layout, where the eye lands first\n" + + "- color: cohesive palette that suits the subject, sufficient contrast\n" + + "- overall: which site a design lead would ship\n" + + "- fit-to-brief: which better delivers what the brief asked for\n\n" + + "Read the HTML and CSS. If a headless browser is available, screenshot both at 1280px and 375px wide and judge the renders. " + + "Judge the sites, not their file names or code style.\n\n" + + `Write \`verdict.json\` in the current directory with exactly the keys ${dims}. ` + + 'Each value is an object: {"winner": "left" | "right" | "tie", "reason": ""}.' + ); +} + +/** One scenario per brief that both treatments have a site for, each with both orderings as variants. */ +export function buildPairwiseScenarios(a: Treatment, b: Treatment, root = ROOT): InlineScenario[] { + const ra = loadReport(a.report, root); + const rb = loadReport(b.report, root); + const scenarios: InlineScenario[] = []; + + for (const result of ra.manifest.results) { + if (result.agentName !== a.agent) continue; + const key = result.scenarioKey; + const other = rb.manifest.results.find((r) => r.scenarioKey === key && r.agentName === b.agent); + const siteA = siteDir(ra.dir, key, a.agent); + const siteB = other && siteDir(rb.dir, key, b.agent); + if (!siteB || !fs.existsSync(path.join(siteA, "index.html")) || !fs.existsSync(path.join(siteB, "index.html"))) { + continue; + } + const copy = (from: string, to: string) => ({ + action: "run_script" as const, + command: `mkdir -p ${to} && cp -R ${shellQuote(from + "/.")} ${to}/`, + }); + scenarios.push({ + key: `pairwise/${key}`, + name: `${result.scenarioName}: A=${a.agent} (${ra.manifest.reportId}) vs B=${b.agent} (${rb.manifest.reportId})`, + prompt: pairwisePrompt(briefOf(result.prompt)), + // Not scored by AXIS (run with --no-score); the verdict file is the output. + judge: "Wrote verdict.json", + artifacts: ["verdict.json"], + variants: [ + { name: "a-left", setup: [copy(siteA, "left"), copy(siteB, "right")] }, + { name: "b-left", setup: [copy(siteB, "left"), copy(siteA, "right")] }, + ], + }); + } + if (scenarios.length === 0) { + throw new Error( + `No scenario has a saved site for both A (${a.agent} in ${a.report}) and B (${b.agent} in ${b.report}).`, + ); + } + return scenarios; +} + +export interface DimensionTally { + a: number; + b: number; + tie: number; + /** Briefs where both orderings named the same winner (or both tied). Low values mean position bias. */ + consistent: number; + pairs: number; +} + +export interface PairwiseSummary { + reportId: string; + dimensions: Record; + /** Runs with no readable verdict.json. */ + missing: string[]; +} + +/** Map each ordering's left/right verdict back to A/B, then pool across briefs. */ +export function summarizePairwise(manifest: ReportManifest, reportDir: string): PairwiseSummary { + const dimensions = Object.fromEntries( + DIMENSIONS.map((d) => [d, { a: 0, b: 0, tie: 0, consistent: 0, pairs: 0 }]), + ) as Record; + const missing: string[] = []; + const byBrief = new Map>>>(); + + for (const result of manifest.results) { + const [base, variant] = result.scenarioKey.split("@") as [string, "a-left" | "b-left"]; + const file = path.join(reportDir, "scenarios", result.scenarioKey, result.agentName, "artifacts", "verdict.json"); + try { + const verdict = JSON.parse(fs.readFileSync(file, "utf-8")); + byBrief.set(base, { ...byBrief.get(base), [variant]: verdict }); + } catch { + missing.push(result.scenarioKey); + } + } + + const toAB = (winner: Side | undefined, aIsLeft: boolean): "a" | "b" | "tie" | undefined => { + if (winner === "tie") return "tie"; + if (winner !== "left" && winner !== "right") return undefined; + return (winner === "left") === aIsLeft ? "a" : "b"; + }; + + for (const orders of byBrief.values()) { + for (const d of DIMENSIONS) { + const first = toAB(orders["a-left"]?.[d]?.winner, true); + const second = toAB(orders["b-left"]?.[d]?.winner, false); + for (const outcome of [first, second]) if (outcome) dimensions[d][outcome]++; + if (first && second) { + dimensions[d].pairs++; + if (first === second) dimensions[d].consistent++; + } + } + } + return { reportId: manifest.reportId, dimensions, missing }; +} + +/** A's win rate with ties counted as half a win. */ +export function winRate(t: DimensionTally): number | undefined { + const n = t.a + t.b + t.tie; + return n === 0 ? undefined : (t.a + t.tie / 2) / n; +} + +export function formatPairwise(summary: PairwiseSummary, title = ""): string { + const rows = DIMENSIONS.map((d) => { + const t = summary.dimensions[d]; + const w = winRate(t); + return [ + d, + String(t.a), + String(t.b), + String(t.tie), + w === undefined ? "" : w.toFixed(2), + t.pairs === 0 ? "" : `${t.consistent}/${t.pairs}`, + ]; + }); + return ( + (title ? `${title}\n\n` : "") + + renderTable(["dimension", "A wins", "B wins", "ties", "A win rate", "order-consistent"], rows) + + (summary.missing.length ? `\nMissing verdicts: ${summary.missing.join(", ")}\n` : "") + + "\nEach brief is judged twice with sides swapped. Win rate counts ties as half.\n" + ); +} + +function shellQuote(s: string): string { + return `'${s.replace(/'/g, `'\\''`)}'`; +} + +function main(argv: string[]): void { + const ref = argv.find((a) => !a.startsWith("--")) ?? "latest"; + const { manifest, dir } = loadReport(ref, ROOT, "pairwise"); + const summary = summarizePairwise(manifest, dir); + const text = formatPairwise(summary, manifest.name ?? `Report ${manifest.reportId}`); + process.stdout.write(argv.includes("--json") ? JSON.stringify(summary, null, 2) + "\n" : text); + if (argv.includes("--write")) { + fs.writeFileSync(path.join(dir, "pairwise.json"), JSON.stringify(summary, null, 2)); + fs.writeFileSync(path.join(dir, "pairwise.md"), `# Pairwise\n\n\`\`\`\n${text}\`\`\`\n`); + } +} + +if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + main(process.argv.slice(2)); +} diff --git a/evals/site-generation/pairwise.config.ts b/evals/site-generation/pairwise.config.ts new file mode 100644 index 0000000..765e388 --- /dev/null +++ b/evals/site-generation/pairwise.config.ts @@ -0,0 +1,25 @@ +import type { AxisConfig } from "../../src/types/index.js"; +import { buildPairwiseScenarios, parseTreatment } from "./lib/pairwise.js"; + +/** + * Pairwise visual judging between two treatments from finished site-generation + * reports. The judge model is the agent here; AXIS scoring is skipped. + * + * PAIRWISE_A=latest:claude-code PAIRWISE_B=latest:codex \ + * npx axis run -c evals/site-generation/pairwise.config.ts --no-score + * + * A and B can come from the same report (Model A vs Model B) or from two + * reports (Config A vs Config B). Win rates land in the pairwise report's + * `pairwise.md`; re-print with `npx jiti evals/site-generation/lib/pairwise.ts`. + */ +export default () => { + const a = parseTreatment(process.env.PAIRWISE_A, "PAIRWISE_A"); + const b = parseTreatment(process.env.PAIRWISE_B, "PAIRWISE_B"); + return { + name: `Pairwise: A=${a.agent} (${a.report}) vs B=${b.agent} (${b.report})`, + scenarios: buildPairwiseScenarios(a, b), + agents: [{ agent: "claude-code", model: process.env.PAIRWISE_JUDGE_MODEL ?? "claude-opus-5-5" }], + settings: { limits: { scenario: { time_minutes: 10 } } }, + afterAll: [{ action: "run_script", command: 'npx jiti lib/pairwise.ts "$AXIS_REPORT_DIR" --write' }], + } satisfies AxisConfig; +}; diff --git a/evals/site-generation/rubric/audits.json b/evals/site-generation/rubric/audits.json new file mode 100644 index 0000000..a7d23f3 --- /dev/null +++ b/evals/site-generation/rubric/audits.json @@ -0,0 +1,9 @@ +{ + "$comment": "Reusable precision audits. A spec lists one by id in `rubric.precision`, or writes its own inline. The judge lists every matching item on the site and labels it SUPPORTED, PLACEHOLDER, or UNSUPPORTED. Precision = supported / (supported + unsupported); placeholders are left out.", + "facts": { + "vertical": "content", + "items": "factual claims: business or person names, contact details, addresses, hours, dates, times, prices, products, services, staff, reviews, testimonials, awards, and statistics", + "supported": "stated in the prompt (or its picker selections)", + "unsupported": "presented as real but not given in the prompt, or contradicting it" + } +} diff --git a/evals/site-generation/rubric/core.json b/evals/site-generation/rubric/core.json new file mode 100644 index 0000000..c744fd2 --- /dev/null +++ b/evals/site-generation/rubric/core.json @@ -0,0 +1,50 @@ +{ + "$comment": "Checks every generated site gets, whatever the scenario. Same shape as a spec's `rubric`: recall items must be present, precision audits count supported vs unsupported items, pass checks are PASS/FAIL.", + "recall": { + "imagery": [ + "At least one image, illustration, icon set, or inline SVG that visibly relates to the subject of the site" + ] + }, + "precision": [ + { + "vertical": "imagery", + "items": "images, illustrations, icons, and decorative graphics", + "supported": "relates to the subject of the site and actually renders (the file exists in `./site/`, or the SVG/CSS is inline)", + "unsupported": "generic filler unrelated to the subject, or a broken or missing image reference" + } + ], + "pass": { + "functional": [ + "`site/index.html` exists and opens directly in a browser without a build step", + "Every local stylesheet, script, and image the site references resolves to a file in `./site/`", + "Every nav link and in-page anchor points at a page or section that exists", + "Every button and form does something: buttons are wired up, and forms have an action or a clearly marked placeholder" + ], + "js-errors": [ + "Scripts are syntactically valid and don't reference undefined elements, functions, or variables. If a headless browser is available, load each page and check the console; otherwise read the scripts. PASS if the site has no JavaScript", + "Interactive behaviour (menu toggles, tabs, carousels, form handling) runs without throwing. PASS if the site has none" + ], + "mobile": [ + "Declares ``", + "At a 375px-wide viewport nothing forces horizontal scrolling: no fixed widths wider than the screen, and images scale down", + "The layout adapts at narrow widths (media queries, flex/grid wrapping, or fluid units) instead of shrinking the desktop layout", + "Navigation stays usable on mobile (collapses, wraps, or stays compact) and tap targets are roughly 44px or well spaced" + ], + "accessibility": [ + "Each page has exactly one `

`, and headings don't skip levels", + "Landmarks are used: `
`, `