feat(ci): run tests and type checks on pull requests

.github/workflows/biome.yml → .github/workflows/code-quality.yml

@@ -29,3 +29,26 @@ jobs:
 
       - name: Check formatting
         run: pnpm format:check
+
+  typecheck:
+    runs-on: ubuntu-26.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@9f698171ed81b15d1823a05fc7211befd50c8ae0 # v6.0.3
+        with:
+          persist-credentials: false
+
+      - name: Install pnpm
+        uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0
+
+      - name: Setup Node.js
+        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
+        with:
+          node-version-file: .node-version
+          cache: pnpm
+
+      - name: Install dependencies
+        run: pnpm install --frozen-lockfile
+
+      - name: Typecheck
+        run: pnpm typecheck

.github/workflows/eval-refresh.yml

@@ -62,17 +62,20 @@ permissions:
   actions: read
 
 concurrency:
-  group: eval-refresh-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.ref }}
+  # The results app's pushes get a unique group so they can't cancel the run that pushed them.
+  group: eval-refresh-${{ github.event.sender.id == 291952723 && github.run_id || github.event_name == 'pull_request' && github.event.pull_request.number || github.ref }}
   # Supersede an in-flight run when a new commit is pushed to the same PR, but
   # never cancel a workflow_dispatch run (those open the refresh PR).
   cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 jobs:
   prepare:
+    # 291952723 is supabase-evals-releaser[bot], which pushes the results commit.
     if: >-
       github.event_name == 'workflow_dispatch' ||
       github.event_name == 'schedule' ||
       (github.event_name == 'pull_request' &&
+        github.event.sender.id != 291952723 &&
         (contains(github.event.pull_request.labels.*.name, 'run-evals') ||
          contains(github.event.pull_request.labels.*.name, 'run-evals-changed')) &&
         github.event.pull_request.head.repo.full_name == github.repository &&
@@ -375,12 +378,8 @@ jobs:
     steps:
       - name: Generate GitHub App token
         id: generate-token
-        # A GitHub App token is needed so the push below can trigger the
-        # gh-pages workflow: GITHUB_TOKEN pushes don't trigger other workflows.
+        # The App token lets the pushes below trigger gh-pages and PR checks.
         # https://docs.github.com/en/actions/how-tos/write-workflows/choose-when-workflows-run/trigger-a-workflow#triggering-a-workflow-from-a-workflow
-        if: >-
-          github.event_name == 'schedule' ||
-          (github.event_name == 'workflow_dispatch' && !inputs.commit_to_branch)
         uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0
         with:
           client-id: ${{ secrets.GH_APP_ID }}
@@ -390,7 +389,7 @@ jobs:
         uses: actions/checkout@9f698171ed81b15d1823a05fc7211befd50c8ae0 # v6.0.3
         with:
           ref: ${{ github.event_name == 'pull_request' && github.head_ref || github.ref }}
-          token: ${{ steps.generate-token.outputs.token || github.token }}
+          token: ${{ steps.generate-token.outputs.token }}
 
       - name: Install pnpm
         uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0
@@ -556,9 +555,8 @@ jobs:
             exit 0
           fi
 
-          # [skip ci] stops the bot push from re-triggering PR workflows (they'd need maintainer approval and loop the refresh):
-          # https://docs.github.com/en/actions/managing-workflow-runs-and-deployments/managing-workflow-runs/skipping-workflow-runs
-          git commit -m "chore: refresh eval results [skip ci]"
+          # This push runs the PR checks. prepare skips it so the refresh doesn't loop.
+          git commit -m "chore: refresh eval results"
           git push
 
       - name: Create results pull request

.github/workflows/test.yml

@@ -0,0 +1,97 @@
+name: Tests
+
+on:
+  pull_request:
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    runs-on: ubuntu-26.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@9f698171ed81b15d1823a05fc7211befd50c8ae0 # v6.0.3
+        with:
+          persist-credentials: false
+
+      - name: Install pnpm
+        uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0
+
+      - name: Setup Node.js
+        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
+        with:
+          node-version-file: .node-version
+          cache: pnpm
+
+      - name: Install dependencies
+        run: pnpm install --frozen-lockfile
+
+      - name: Run tests
+        # The framework smoke test calls an LLM judge. This is expected to fail on fork PRs.
+        env:
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+        run: pnpm test
+
+  # Separate job so it runs in parallel with `test`.
+  test-platform-lite:
+    runs-on: ubuntu-26.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@9f698171ed81b15d1823a05fc7211befd50c8ae0 # v6.0.3
+        with:
+          persist-credentials: false
+
+      - name: Install pnpm
+        uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0
+
+      - name: Setup Node.js
+        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
+        with:
+          node-version-file: .node-version
+          cache: pnpm
+
+      - name: Install dependencies
+        run: pnpm install --frozen-lockfile
+
+      - name: Run platform-lite tests
+        run: pnpm --filter @supabase-evals/platform-lite test
+
+  # Slow, so the steps only run when the sandbox changes. A `paths:` filter would leave the required check pending.
+  test-docker:
+    runs-on: ubuntu-26.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@9f698171ed81b15d1823a05fc7211befd50c8ae0 # v6.0.3
+        with:
+          persist-credentials: false
+          # HEAD is the PR merge commit and HEAD^1 is the base, so their diff is the PR's changes.
+          fetch-depth: 2
+
+      - name: Check for sandbox changes
+        id: changes
+        run: |
+          if git diff --quiet HEAD^1 HEAD -- packages/sandbox pnpm-lock.yaml .github/workflows/test.yml; then
+            echo "run=false" >> "$GITHUB_OUTPUT"
+          else
+            echo "run=true" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Install pnpm
+        if: steps.changes.outputs.run == 'true'
+        uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0
+
+      - name: Setup Node.js
+        if: steps.changes.outputs.run == 'true'
+        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
+        with:
+          node-version-file: .node-version
+          cache: pnpm
+
+      - name: Install dependencies
+        if: steps.changes.outputs.run == 'true'
+        run: pnpm install --frozen-lockfile
+
+      - name: Run Docker integration tests
+        if: steps.changes.outputs.run == 'true'
+        run: pnpm --filter @supabase-evals/sandbox test:docker

CONTRIBUTING.md

@@ -96,4 +96,4 @@ Common workflows:
 - **Add or change a CLI eval.** Add the scenario under `evals/cli/<id>/` (see [Adding an eval](#adding-an-eval)); set `needsDocker: false` in its `PROMPT.md` frontmatter if it can run without Docker, open a PR, and add the `run-evals-changed` label. Results for the changed evals are committed back to your branch and viewable in the Vercel preview.
 - **Refresh every CLI eval.** Dispatch the [Refresh eval results](https://github.com/supabase/evals/actions/workflows/eval-refresh.yml) workflow on `main` with `suite: cli` and `experiment_suite: cli`. It opens a draft PR with the updated `cli-eval-results.json` for you to review and merge. Leave `cli_stable_version`/`cli_beta_version` blank to resolve npm's latest dist-tags, or pin them to reproduce a specific run.
 - **Analyze results over time.** Every merge that changes `cli-eval-results.json` appends a snapshot to [`cli-results.jsonl`](https://supabase.github.io/evals/cli-results.jsonl) on GitHub Pages, alongside the [benchmark](https://supabase.github.io/evals/results.jsonl), [regression](https://supabase.github.io/evals/regression-results.jsonl), and [docs](https://supabase.github.io/evals/docs-results.jsonl) histories.
-- **Run the unit tests.** `pnpm --filter @supabase-evals/framework test:cli-lib` (the CLI skip predicates in `experiments/cli/lib/` plus every CLI eval's scorer tests) — also part of `pnpm check`.
+- **Run the unit tests.** `pnpm --filter @supabase-evals/framework test:cli-lib` (the CLI skip predicates in `experiments/cli/lib/` plus every CLI eval's scorer tests) — also part of `pnpm test`.

README.md

@@ -151,7 +151,7 @@ An experiment can pass `localStackRuntime({ docker: 'no-daemon' })` or `'absent'
 
 Scorers check what the agent produced, never what the harness provisioned: with `projectRunning: true` (the default) the running stack and the seeded `local/` workspace are setup, so score only the deltas the agent made on top; with `projectRunning: false` the agent creates that state itself, so depending on it is fair game.
 
-Test the sandbox plumbing without an agent run (Docker required, not part of `pnpm check`):
+Test the sandbox plumbing without an agent run (Docker required, not part of `pnpm test`):
 
 ```bash
 pnpm --filter @supabase-evals/sandbox test:docker
@@ -168,10 +168,11 @@ Skills are installed into the sandbox workspace with [Vercel's `skills` CLI](htt
 ## Framework Checks
 
 ```bash
-pnpm check
+pnpm typecheck
+pnpm test
 ```
 
-Runs typechecks plus local smoke tests.
+Typechecks every package, then runs the local tests.
 
 ## Contributing
 

apps/framework/package.json

@@ -4,17 +4,16 @@
   "version": "0.0.1",
   "type": "module",
   "scripts": {
-    "check": "pnpm typecheck && pnpm test && pnpm test:framework && pnpm test:cli-lib",
     "eval": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts",
     "eval:upload": "node --import tsx/esm scripts/eval-upload.ts",
     "eval:dry": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts --dry",
     "eval:smoke": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts --smoke",
     "eval:vercel": "node --env-file=../../.env --import tsx/esm scripts/run-vercel-evals.ts",
     "typecheck": "tsc --noEmit",
-    "test": "vitest run",
+    "test": "vitest run && pnpm test:framework && pnpm test:cli-lib",
     "test:framework": "node --env-file-if-exists=../../.env --import tsx/esm scripts/smoke-framework.ts",
     "test:vercel-runner": "vitest run scripts/run-vercel-evals.test.ts scripts/export-results.test.ts lib/cli-args.test.ts lib/experiment-files.test.ts lib/sample-sets.test.ts",
-    "test:cli-lib": "vitest run --root ../.. --passWithNoTests experiments/cli evals/cli",
+    "test:cli-lib": "vitest run --config vitest.cli-lib.config.ts --passWithNoTests experiments/cli evals/cli",
     "export-results": "node --import tsx/esm scripts/export-results.ts",
     "demo:mcp": "node --env-file=../../.env --import tsx/esm scripts/mcp-demo.ts",
     "demo:executor": "node --env-file=../../.env --import tsx/esm scripts/executor-demo.ts",

apps/framework/vitest.cli-lib.config.ts

@@ -0,0 +1,3 @@
+import { defineConfig } from 'vitest/config';
+
+export default defineConfig({ test: { name: 'cli evals', root: '../..' } });

apps/framework/vitest.config.ts

@@ -0,0 +1,3 @@
+import { defineConfig } from 'vitest/config';
+
+export default defineConfig({ test: { name: 'framework' } });

apps/web/vite.config.ts

@@ -1,3 +1,4 @@
+/// <reference types="vitest/config" />
 import path from "path"
 import tailwindcss from "@tailwindcss/vite"
 import react from "@vitejs/plugin-react"
@@ -21,4 +22,7 @@ export default defineConfig({
       "@": path.resolve(__dirname, "./src"),
     },
   },
+  test: {
+    name: "web",
+  },
 })

evals/docs/.claude/skills/docs-eval-planning/references/flakiness.md

@@ -157,7 +157,7 @@ Add to this file. A documentation eval is not finished until whatever went wrong
 - **Ports.** The range is 54321 to 54329. Another project's stack holding them blocked local runs
   entirely, and one eval shipped for review having never run end to end because of it. #168, #206
   **Fix.** Preflight the ports. Both later evals open their testing steps with it.
-- **`pnpm check` needs `OPENAI_API_KEY`** and stops without it, which reads as a broken diff.
+- **`pnpm test` needs `OPENAI_API_KEY`** and stops without it, which reads as a broken diff.
   #228, #168
 - **A stale local `VERCEL_OIDC_TOKEN` shadows the real one** and every sandbox create fails. #192
 - **`ctx.stackStatus()` requires three values together** and throws when any is missing, so an eval

package.json

@@ -4,7 +4,7 @@
   "type": "module",
   "workspaces": ["apps/*", "packages/*"],
   "scripts": {
-    "check": "pnpm --filter @supabase-evals/framework check && pnpm --filter @supabase-evals/core test && pnpm --filter @supabase-evals/sandbox test && pnpm --filter @supabase-evals/web test",
+    "test": "pnpm --filter @supabase-evals/framework test && pnpm --filter @supabase-evals/core test && pnpm --filter @supabase-evals/sandbox test && pnpm --filter @supabase-evals/web test",
     "eval": "pnpm --filter @supabase-evals/framework eval",
     "eval:dry": "pnpm --filter @supabase-evals/framework eval:dry",
     "eval:smoke": "pnpm --filter @supabase-evals/framework eval:smoke",
@@ -13,7 +13,7 @@
     "test:framework": "pnpm --filter @supabase-evals/framework test:framework",
     "export-results": "pnpm --filter @supabase-evals/framework export-results",
     "upload-braintrust": "pnpm --filter @supabase-evals/framework upload-braintrust",
-    "typecheck": "pnpm --filter @supabase-evals/framework typecheck && pnpm --filter @supabase-evals/web typecheck",
+    "typecheck": "pnpm -r typecheck",
     "web": "pnpm --filter @supabase-evals/web dev",
     "web:build": "pnpm --filter @supabase-evals/web build",
     "demo:mcp": "pnpm --filter @supabase-evals/framework demo:mcp",

packages/core/vitest.config.ts

@@ -0,0 +1,3 @@
+import { defineConfig } from 'vitest/config';
+
+export default defineConfig({ test: { name: 'core' } });

packages/platform-lite/vitest.config.ts

@@ -2,6 +2,7 @@ import { defineConfig } from 'vitest/config';
 
 export default defineConfig({
   test: {
+    name: 'platform-lite',
     environment: 'node',
     testTimeout: 30000,
   },

packages/sandbox/vitest.config.ts

@@ -0,0 +1,3 @@
+import { defineConfig } from 'vitest/config';
+
+export default defineConfig({ test: { name: 'sandbox' } });