From 178b7052d3e23d7faff4363b4d3f1e504bb9a461 Mon Sep 17 00:00:00 2001
From: Matt Rossman <22670878+mattrossman@users.noreply.github.com>
Date: Fri, 2 Oct 2026 17:27:59 -0400
Subject: [PATCH 1/8] fix(ci): support mixed explicit refresh experiments

---
 .github/workflows/eval-refresh.yml | 21 ++++++++++++++-------
 1 file changed, 14 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/eval-refresh.yml b/.github/workflows/eval-refresh.yml
index fc7833f8..c858e77e 100644
--- a/.github/workflows/eval-refresh.yml
+++ b/.github/workflows/eval-refresh.yml
@@ -16,7 +16,7 @@ on:
         required: true
         default: "benchmark"
       experiment_suite:
-        description: "Comma-separated experiment suites to run"
+        description: "Comma-separated experiment suites to run (ignored when experiments is set)"
         required: true
         default: "benchmark,no-skills"
       runs:
@@ -235,7 +235,7 @@ jobs:
           fi
 
           pairs='[]'
-          experiments_override="${{ steps.inputs.outputs.experiments_override }}"
+          experiments_override="$(jq -Rc 'split(",") | map(gsub("^\\s+|\\s+$"; "")) | map(select(length > 0))' <<< "${{ steps.inputs.outputs.experiments_override }}")"
           for id in "${matching[@]}"; do
             eval_suite=$(basename "$(dirname "$(compgen -G "evals/*/$id")")")
             case "$eval_suite" in
@@ -247,15 +247,16 @@ jobs:
             esac
 
             for experiment_suite in "${experiment_suites[@]}"; do
-              if ! jq -e --arg suite "$experiment_suite" 'index($suite) != null' \
+              # experiment_suite only narrows auto-discovery. An explicit list
+              # runs each experiment in whichever of these suites it belongs to.
+              if [ "$experiments_override" = "[]" ] && ! jq -e --arg suite "$experiment_suite" 'index($suite) != null' \
                 <<< '${{ steps.inputs.outputs.experiment_suite }}' > /dev/null; then
                 continue
               fi
 
-              if [ -n "$experiments_override" ]; then
-                experiments_json="$(jq -Rc 'split(",") | map(gsub("^\\s+|\\s+$"; "")) | map(select(length > 0))' <<< "$experiments_override")"
-              else
-                experiments_json="$(pnpm --silent eval -- list --experiment-suite "$experiment_suite" --eval "$id")"
+              experiments_json="$(pnpm --silent eval -- list --experiment-suite "$experiment_suite" --eval "$id")"
+              if [ "$experiments_override" != "[]" ]; then
+                experiments_json="$(jq -c --argjson want "$experiments_override" 'map(select(IN($want[])))' <<< "$experiments_json")"
               fi
 
               while IFS= read -r experiment; do
@@ -270,6 +271,12 @@ jobs:
             done
           done
 
+          unmatched="$(jq -r --argjson pairs "$pairs" '. - ($pairs | map(.experiment)) | join(",")' <<< "$experiments_override")"
+          if [ -n "$unmatched" ]; then
+            echo "Experiments not in any suite for the selected evals: $unmatched" >&2
+            exit 1
+          fi
+
           if [ "$(jq 'length' <<< "$pairs")" -eq 0 ]; then
             echo "No experiment and eval pairs matched" >&2
             exit 1

From 667364cf3460bb0f6c9a343ce84e57ef3d6df3d9 Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Fri, 2 Oct 2026 21:35:01 +0000
Subject: [PATCH 2/8] chore: refresh eval results [skip ci]

---
 apps/web/src/data/eval-results.json | 536 +++-------------------------
 1 file changed, 57 insertions(+), 479 deletions(-)

diff --git a/apps/web/src/data/eval-results.json b/apps/web/src/data/eval-results.json
index ce7cde21..7d47a3b7 100644
--- a/apps/web/src/data/eval-results.json
+++ b/apps/web/src/data/eval-results.json
@@ -20,6 +20,7 @@
     ],
     "suite": "benchmark",
     "interface": "cli",
+    "cliVersion": "2.117.0",
     "passed": true,
     "checks": [
       {
@@ -30,7 +31,7 @@
       {
         "name": "signUp creates the account and returns its user id",
         "passed": true,
-        "notes": "db user fd460ee4-c88d-4bb2-b257-cc25ec7d7336, signUp returned {\"userId\":\"fd460ee4-c88d-4bb2-b257-cc25ec7d7336\"}"
+        "notes": "db user 18650c46-b81c-4c03-95a0-b4a12df0ec37, signUp returned {\"userId\":\"18650c46-b81c-4c03-95a0-b4a12df0ec37\"}"
       },
       {
         "name": "signup metadata reaches the profile (display name)",
@@ -45,7 +46,7 @@
       {
         "name": "signIn with the right password returns the user id",
         "passed": true,
-        "notes": "{\"userId\":\"fd460ee4-c88d-4bb2-b257-cc25ec7d7336\"}"
+        "notes": "{\"userId\":\"18650c46-b81c-4c03-95a0-b4a12df0ec37\"}"
       },
       {
         "name": "getMyProfile returns the signed-in user's profile",
@@ -78,215 +79,29 @@
     "usage": [
       {
         "model": "claude-opus-5-5",
-        "inputTokens": 303109,
-        "cacheReadInputTokens": 280917,
-        "cacheWriteInputTokens": 22170,
-        "outputTokens": 4981
+        "inputTokens": 221737,
+        "cacheReadInputTokens": 199328,
+        "cacheWriteInputTokens": 22393,
+        "outputTokens": 5412
       }
     ],
-    "stepCount": 11,
-    "toolCallCount": 10,
-    "agentRunDurationMs": 68653,
+    "stepCount": 10,
+    "toolCallCount": 8,
+    "agentRunDurationMs": 71028,
     "sandboxUsage": {
-      "memory": 8192
+      "activeCpuDurationMs": 254534,
+      "duration": 217158,
+      "memory": 8192,
+      "networkTransfer": {
+        "ingress": 1597805842,
+        "egress": 7134627
+      }
     },
     "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.",
     "promptSourcePath": "evals/benchmark/build-auth-001-email-password-flow/PROMPT.md",
     "run": 1,
     "sourcePath": "claude-code-opus-5.5/build-auth-001-email-password-flow/run-1/result.json"
   },
-  {
-    "experiment": "claude-code-opus-5.5",
-    "experimentSuite": "benchmark",
-    "experimentDisplay": {
-      "agent": "claude-code",
-      "modelProvider": "anthropic",
-      "modelId": "claude-opus-5-5",
-      "reasoningEffort": "high"
-    },
-    "eval": "build-auth-001-email-password-flow",
-    "stage": "build",
-    "product": [
-      "auth",
-      "database"
-    ],
-    "topic": [
-      "sdk",
-      "rls"
-    ],
-    "suite": "benchmark",
-    "interface": "cli",
-    "passed": true,
-    "checks": [
-      {
-        "name": "auth module loads and the driver completes",
-        "passed": true,
-        "notes": "driver produced a result"
-      },
-      {
-        "name": "signUp creates the account and returns its user id",
-        "passed": true,
-        "notes": "db user 404fbc6e-816b-4faf-bda1-e2b01b5f46d1, signUp returned {\"userId\":\"404fbc6e-816b-4faf-bda1-e2b01b5f46d1\"}"
-      },
-      {
-        "name": "signup metadata reaches the profile (display name)",
-        "passed": true,
-        "notes": "profiles.display_name = \"Alex Doe\""
-      },
-      {
-        "name": "wrong password is rejected gracefully (no throw, no session)",
-        "passed": true,
-        "notes": "{\"error\":\"Invalid login credentials\"}"
-      },
-      {
-        "name": "signIn with the right password returns the user id",
-        "passed": true,
-        "notes": "{\"userId\":\"404fbc6e-816b-4faf-bda1-e2b01b5f46d1\"}"
-      },
-      {
-        "name": "getMyProfile returns the signed-in user's profile",
-        "passed": true,
-        "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}"
-      },
-      {
-        "name": "app code does not use the secret / service-role key",
-        "passed": true,
-        "notes": "no secret-key references found"
-      },
-      {
-        "name": "implementation uses @supabase/supabase-js",
-        "passed": true,
-        "notes": "imports found in: app/src/auth.mjs"
-      }
-    ],
-    "skills": {
-      "available": [
-        "supabase",
-        "supabase-postgres-best-practices"
-      ],
-      "loaded": [
-        "supabase"
-      ]
-    },
-    "docs": {
-      "calls": []
-    },
-    "usage": [
-      {
-        "model": "claude-opus-5-5",
-        "inputTokens": 192261,
-        "cacheReadInputTokens": 173763,
-        "cacheWriteInputTokens": 18482,
-        "outputTokens": 4361
-      }
-    ],
-    "stepCount": 8,
-    "toolCallCount": 7,
-    "agentRunDurationMs": 55604,
-    "sandboxUsage": {
-      "memory": 8192
-    },
-    "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.",
-    "promptSourcePath": "evals/benchmark/build-auth-001-email-password-flow/PROMPT.md",
-    "run": 2,
-    "sourcePath": "claude-code-opus-5.5/build-auth-001-email-password-flow/run-2/result.json"
-  },
-  {
-    "experiment": "claude-code-opus-5.5",
-    "experimentSuite": "benchmark",
-    "experimentDisplay": {
-      "agent": "claude-code",
-      "modelProvider": "anthropic",
-      "modelId": "claude-opus-5-5",
-      "reasoningEffort": "high"
-    },
-    "eval": "build-auth-001-email-password-flow",
-    "stage": "build",
-    "product": [
-      "auth",
-      "database"
-    ],
-    "topic": [
-      "sdk",
-      "rls"
-    ],
-    "suite": "benchmark",
-    "interface": "cli",
-    "passed": true,
-    "checks": [
-      {
-        "name": "auth module loads and the driver completes",
-        "passed": true,
-        "notes": "driver produced a result"
-      },
-      {
-        "name": "signUp creates the account and returns its user id",
-        "passed": true,
-        "notes": "db user 710f55b4-b191-4c4a-9a55-b24eb98efa01, signUp returned {\"userId\":\"710f55b4-b191-4c4a-9a55-b24eb98efa01\"}"
-      },
-      {
-        "name": "signup metadata reaches the profile (display name)",
-        "passed": true,
-        "notes": "profiles.display_name = \"Alex Doe\""
-      },
-      {
-        "name": "wrong password is rejected gracefully (no throw, no session)",
-        "passed": true,
-        "notes": "{\"error\":\"Invalid login credentials\"}"
-      },
-      {
-        "name": "signIn with the right password returns the user id",
-        "passed": true,
-        "notes": "{\"userId\":\"710f55b4-b191-4c4a-9a55-b24eb98efa01\"}"
-      },
-      {
-        "name": "getMyProfile returns the signed-in user's profile",
-        "passed": true,
-        "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}"
-      },
-      {
-        "name": "app code does not use the secret / service-role key",
-        "passed": true,
-        "notes": "no secret-key references found"
-      },
-      {
-        "name": "implementation uses @supabase/supabase-js",
-        "passed": true,
-        "notes": "imports found in: app/src/auth.mjs"
-      }
-    ],
-    "skills": {
-      "available": [
-        "supabase",
-        "supabase-postgres-best-practices"
-      ],
-      "loaded": [
-        "supabase"
-      ]
-    },
-    "docs": {
-      "calls": []
-    },
-    "usage": [
-      {
-        "model": "claude-opus-5-5",
-        "inputTokens": 231202,
-        "cacheReadInputTokens": 211534,
-        "cacheWriteInputTokens": 19650,
-        "outputTokens": 4820
-      }
-    ],
-    "stepCount": 11,
-    "toolCallCount": 9,
-    "agentRunDurationMs": 60907,
-    "sandboxUsage": {
-      "memory": 8192
-    },
-    "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.",
-    "promptSourcePath": "evals/benchmark/build-auth-001-email-password-flow/PROMPT.md",
-    "run": 3,
-    "sourcePath": "claude-code-opus-5.5/build-auth-001-email-password-flow/run-3/result.json"
-  },
   {
     "experiment": "claude-code-opus-5.5",
     "experimentSuite": "benchmark",
@@ -38900,6 +38715,7 @@
     ],
     "suite": "benchmark",
     "interface": "cli",
+    "cliVersion": "2.117.0",
     "passed": true,
     "checks": [
       {
@@ -38910,7 +38726,7 @@
       {
         "name": "signUp creates the account and returns its user id",
         "passed": true,
-        "notes": "db user 1907b8d1-8968-4adb-9356-6ffa937bafb2, signUp returned {\"userId\":\"1907b8d1-8968-4adb-9356-6ffa937bafb2\"}"
+        "notes": "db user ad9d26de-db58-4478-a1f7-d6422990540c, signUp returned {\"userId\":\"ad9d26de-db58-4478-a1f7-d6422990540c\"}"
       },
       {
         "name": "signup metadata reaches the profile (display name)",
@@ -38925,7 +38741,7 @@
       {
         "name": "signIn with the right password returns the user id",
         "passed": true,
-        "notes": "{\"userId\":\"1907b8d1-8968-4adb-9356-6ffa937bafb2\"}"
+        "notes": "{\"userId\":\"ad9d26de-db58-4478-a1f7-d6422990540c\"}"
       },
       {
         "name": "getMyProfile returns the signed-in user's profile",
@@ -38951,342 +38767,104 @@
       "calls": [
         {
           "source": "search_docs",
-          "query": "query { searchDocs(query: \"Supabase JavaScript auth signUp signInWithPassword user metadata profile table RLS\", limit: 5) { nodes { title href content } } }",
+          "query": "query { searchDocs(query: \"Supabase JavaScript sign up sign in password user metadata auth getUser select profile RLS\", limit: 6) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content methodName } } } }",
           "hasContent": true,
           "pages": [
             {
-              "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security",
-              "title": "Token Security and Row Level Security"
-            },
-            {
-              "url": "https://supabase.com/docs/guides/database/secure-data",
-              "title": "Securing your data"
+              "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0",
+              "title": "Migrate from Auth0 to Supabase Auth"
             },
             {
               "url": "https://supabase.com/docs/guides/auth/users",
               "title": "Users"
             },
             {
-              "url": "https://supabase.com/docs/guides/api/securing-your-api",
-              "title": "Securing your API"
+              "url": "https://supabase.com/docs/guides/auth",
+              "title": "Auth"
             },
             {
               "url": "https://supabase.com/docs/guides/auth/third-party/firebase-auth",
               "title": "Firebase Auth"
+            },
+            {
+              "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security",
+              "title": "Token Security and Row Level Security"
             }
           ],
-          "resultChars": 66949
+          "resultChars": 59482
         },
         {
           "source": "search_docs",
-          "query": "query { searchDocs(query: \"supabase-js createClient auth.signUp options data user_metadata signInWithPassword select single\", limit: 5) { nodes { title href content } } }",
+          "query": "query { searchDocs(query: \"supabase-js auth signUp options data user_metadata signInWithPassword getUser\", limit: 5) { nodes { ... on ClientLibraryFunctionReference { title href content methodName language } ... on Guide { title href content } } } }",
           "hasContent": true,
           "pages": [
             {
-              "url": "https://supabase.com/docs/guides/auth/passwords",
-              "title": "Password-based Auth"
+              "url": "https://supabase.com/docs/guides/auth/managing-user-data",
+              "title": "User Management"
             },
             {
               "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0",
               "title": "Migrate from Auth0 to Supabase Auth"
             },
             {
-              "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso",
-              "title": "Configure SAML SSO"
+              "url": "https://supabase.com/docs/guides/auth/social-login/auth-google",
+              "title": "Sign in with Google"
             },
             {
               "url": "https://supabase.com/docs/guides/auth/passkeys",
               "title": "Passkey authentication"
-            },
-            {
-              "url": "https://supabase.com/docs/guides/troubleshooting/customizing-emails-by-language-KZ_38Q",
-              "title": "Customizing Emails by Language"
             }
           ],
-          "resultChars": 101691
-        }
-      ]
-    },
-    "usage": [
-      {
-        "model": "gpt-6-luna",
-        "inputTokens": 255262,
-        "cacheReadInputTokens": 210632,
-        "cacheWriteInputTokens": 43882,
-        "outputTokens": 9832
-      }
-    ],
-    "stepCount": 11,
-    "toolCallCount": 14,
-    "agentRunDurationMs": 99827,
-    "sandboxUsage": {
-      "memory": 8192
-    },
-    "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.",
-    "promptSourcePath": "evals/benchmark/build-auth-001-email-password-flow/PROMPT.md",
-    "run": 1,
-    "sourcePath": "codex-gpt-6-luna-no-skills/build-auth-001-email-password-flow/run-1/result.json"
-  },
-  {
-    "experiment": "codex-gpt-6-luna-no-skills",
-    "experimentSuite": "no-skills",
-    "experimentDisplay": {
-      "agent": "codex",
-      "modelProvider": "openai",
-      "modelId": "gpt-6-luna",
-      "reasoningEffort": "medium"
-    },
-    "eval": "build-auth-001-email-password-flow",
-    "stage": "build",
-    "product": [
-      "auth",
-      "database"
-    ],
-    "topic": [
-      "sdk",
-      "rls"
-    ],
-    "suite": "benchmark",
-    "interface": "cli",
-    "passed": true,
-    "checks": [
-      {
-        "name": "auth module loads and the driver completes",
-        "passed": true,
-        "notes": "driver produced a result"
-      },
-      {
-        "name": "signUp creates the account and returns its user id",
-        "passed": true,
-        "notes": "db user c461532d-f457-48de-a404-d3a7b7ee3d6c, signUp returned {\"userId\":\"c461532d-f457-48de-a404-d3a7b7ee3d6c\"}"
-      },
-      {
-        "name": "signup metadata reaches the profile (display name)",
-        "passed": true,
-        "notes": "profiles.display_name = \"Alex Doe\""
-      },
-      {
-        "name": "wrong password is rejected gracefully (no throw, no session)",
-        "passed": true,
-        "notes": "{\"error\":\"Invalid login credentials\"}"
-      },
-      {
-        "name": "signIn with the right password returns the user id",
-        "passed": true,
-        "notes": "{\"userId\":\"c461532d-f457-48de-a404-d3a7b7ee3d6c\"}"
-      },
-      {
-        "name": "getMyProfile returns the signed-in user's profile",
-        "passed": true,
-        "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}"
-      },
-      {
-        "name": "app code does not use the secret / service-role key",
-        "passed": true,
-        "notes": "no secret-key references found"
-      },
-      {
-        "name": "implementation uses @supabase/supabase-js",
-        "passed": true,
-        "notes": "imports found in: app/src/auth.mjs"
-      }
-    ],
-    "skills": {
-      "available": [],
-      "loaded": []
-    },
-    "docs": {
-      "calls": [
+          "resultChars": 84771
+        },
         {
           "source": "search_docs",
-          "query": "query { searchDocs(query: \"JavaScript auth signUp signInWithPassword getUser user metadata profiles table\", limit: 5) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content methodName } } } }",
+          "query": "query { searchDocs(query: \"JavaScript auth signInWithPassword auth getUser createClient persist session local storage\", limit: 4) { nodes { ... on ClientLibraryFunctionReference { title href content methodName } ... on Guide { title href content } } } }",
           "hasContent": true,
           "pages": [
-            {
-              "url": "https://supabase.com/docs/guides/auth/managing-user-data",
-              "title": "User Management"
-            },
             {
               "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpassword"
             },
             {
-              "url": "https://supabase.com/docs/reference/swift/auth-signinwithpassword",
-              "title": "signInWithPassword()"
+              "url": "https://supabase.com/docs/guides/auth/passwords",
+              "title": "Password-based Auth"
             },
             {
               "url": "https://supabase.com/docs/reference/dart/auth-signinwithpassword"
             },
             {
-              "url": "https://supabase.com/docs/reference/javascript/auth-signup"
+              "url": "https://supabase.com/docs/reference/javascript/auth-signinwithoauth"
             }
           ],
-          "resultChars": 11371
+          "resultChars": 50717
         }
       ]
     },
     "usage": [
       {
         "model": "gpt-6-luna",
-        "inputTokens": 316681,
-        "cacheReadInputTokens": 271695,
-        "cacheWriteInputTokens": 44034,
-        "outputTokens": 10981
+        "inputTokens": 680305,
+        "cacheReadInputTokens": 615992,
+        "cacheWriteInputTokens": 62885,
+        "outputTokens": 21537
       }
     ],
-    "stepCount": 14,
-    "toolCallCount": 20,
-    "agentRunDurationMs": 124141,
+    "stepCount": 21,
+    "toolCallCount": 25,
+    "agentRunDurationMs": 210872,
     "sandboxUsage": {
-      "memory": 8192
-    },
-    "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.",
-    "promptSourcePath": "evals/benchmark/build-auth-001-email-password-flow/PROMPT.md",
-    "run": 2,
-    "sourcePath": "codex-gpt-6-luna-no-skills/build-auth-001-email-password-flow/run-2/result.json"
-  },
-  {
-    "experiment": "codex-gpt-6-luna-no-skills",
-    "experimentSuite": "no-skills",
-    "experimentDisplay": {
-      "agent": "codex",
-      "modelProvider": "openai",
-      "modelId": "gpt-6-luna",
-      "reasoningEffort": "medium"
-    },
-    "eval": "build-auth-001-email-password-flow",
-    "stage": "build",
-    "product": [
-      "auth",
-      "database"
-    ],
-    "topic": [
-      "sdk",
-      "rls"
-    ],
-    "suite": "benchmark",
-    "interface": "cli",
-    "passed": true,
-    "checks": [
-      {
-        "name": "auth module loads and the driver completes",
-        "passed": true,
-        "notes": "driver produced a result"
-      },
-      {
-        "name": "signUp creates the account and returns its user id",
-        "passed": true,
-        "notes": "db user db249555-d311-45e6-9a5e-4380aaf5ce1f, signUp returned {\"userId\":\"db249555-d311-45e6-9a5e-4380aaf5ce1f\"}"
-      },
-      {
-        "name": "signup metadata reaches the profile (display name)",
-        "passed": true,
-        "notes": "profiles.display_name = \"Alex Doe\""
-      },
-      {
-        "name": "wrong password is rejected gracefully (no throw, no session)",
-        "passed": true,
-        "notes": "{\"error\":\"Invalid login credentials\"}"
-      },
-      {
-        "name": "signIn with the right password returns the user id",
-        "passed": true,
-        "notes": "{\"userId\":\"db249555-d311-45e6-9a5e-4380aaf5ce1f\"}"
-      },
-      {
-        "name": "getMyProfile returns the signed-in user's profile",
-        "passed": true,
-        "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}"
-      },
-      {
-        "name": "app code does not use the secret / service-role key",
-        "passed": true,
-        "notes": "no secret-key references found"
-      },
-      {
-        "name": "implementation uses @supabase/supabase-js",
-        "passed": true,
-        "notes": "imports found in: app/src/auth.mjs"
-      }
-    ],
-    "skills": {
-      "available": [],
-      "loaded": []
-    },
-    "docs": {
-      "calls": [
-        {
-          "source": "search_docs",
-          "query": "query { searchDocs(query: \"Supabase JavaScript auth signUp signInWithPassword getUser user metadata profile table Row Level Security\", limit: 5) { nodes { title href content } } }",
-          "hasContent": true,
-          "pages": [
-            {
-              "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt",
-              "title": "Integrating With Supabase Auth"
-            },
-            {
-              "url": "https://supabase.com/docs/guides/auth",
-              "title": "Auth"
-            },
-            {
-              "url": "https://supabase.com/docs/guides/database/secure-data",
-              "title": "Securing your data"
-            },
-            {
-              "url": "https://supabase.com/docs/guides/storage/security/access-control",
-              "title": "Storage Access Control"
-            },
-            {
-              "url": "https://supabase.com/docs/guides/deployment/going-into-prod",
-              "title": "Production Checklist"
-            }
-          ],
-          "resultChars": 32006
-        },
-        {
-          "source": "search_docs",
-          "query": "query { searchDocs(query: \"supabase-js createClient auth signUp options data signInWithPassword getUser javascript\", limit: 4) { nodes { title href content } } }",
-          "hasContent": true,
-          "pages": [
-            {
-              "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0",
-              "title": "Migrate from Auth0 to Supabase Auth"
-            },
-            {
-              "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso",
-              "title": "Configure SAML SSO"
-            },
-            {
-              "url": "https://supabase.com/docs/guides/auth/social-login/auth-figma",
-              "title": "Sign in with Figma"
-            },
-            {
-              "url": "https://supabase.com/docs/guides/auth/passwords",
-              "title": "Password-based Auth"
-            }
-          ],
-          "resultChars": 104607
-        }
-      ]
-    },
-    "usage": [
-      {
-        "model": "gpt-6-luna",
-        "inputTokens": 295727,
-        "cacheReadInputTokens": 251699,
-        "cacheWriteInputTokens": 43144,
-        "outputTokens": 8956
+      "activeCpuDurationMs": 271509,
+      "duration": 358020,
+      "memory": 8192,
+      "networkTransfer": {
+        "ingress": 1656125723,
+        "egress": 12470800
       }
-    ],
-    "stepCount": 13,
-    "toolCallCount": 15,
-    "agentRunDurationMs": 89961,
-    "sandboxUsage": {
-      "memory": 8192
     },
     "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.",
     "promptSourcePath": "evals/benchmark/build-auth-001-email-password-flow/PROMPT.md",
-    "run": 3,
-    "sourcePath": "codex-gpt-6-luna-no-skills/build-auth-001-email-password-flow/run-3/result.json"
+    "run": 1,
+    "sourcePath": "codex-gpt-6-luna-no-skills/build-auth-001-email-password-flow/run-1/result.json"
   },
   {
     "experiment": "codex-gpt-6-luna-no-skills",

From 5f22ed4fa9ed3f3805a685b886520ada86378fac Mon Sep 17 00:00:00 2001
From: Matt Rossman <22670878+mattrossman@users.noreply.github.com>
Date: Fri, 2 Oct 2026 17:35:38 -0400
Subject: [PATCH 3/8] Revert "chore: refresh eval results [skip ci]"

This reverts commit 667364cf3460bb0f6c9a343ce84e57ef3d6df3d9.
---
 apps/web/src/data/eval-results.json | 536 +++++++++++++++++++++++++---
 1 file changed, 479 insertions(+), 57 deletions(-)

diff --git a/apps/web/src/data/eval-results.json b/apps/web/src/data/eval-results.json
index 7d47a3b7..ce7cde21 100644
--- a/apps/web/src/data/eval-results.json
+++ b/apps/web/src/data/eval-results.json
@@ -20,7 +20,6 @@
     ],
     "suite": "benchmark",
     "interface": "cli",
-    "cliVersion": "2.117.0",
     "passed": true,
     "checks": [
       {
@@ -31,7 +30,7 @@
       {
         "name": "signUp creates the account and returns its user id",
         "passed": true,
-        "notes": "db user 18650c46-b81c-4c03-95a0-b4a12df0ec37, signUp returned {\"userId\":\"18650c46-b81c-4c03-95a0-b4a12df0ec37\"}"
+        "notes": "db user fd460ee4-c88d-4bb2-b257-cc25ec7d7336, signUp returned {\"userId\":\"fd460ee4-c88d-4bb2-b257-cc25ec7d7336\"}"
       },
       {
         "name": "signup metadata reaches the profile (display name)",
@@ -46,7 +45,7 @@
       {
         "name": "signIn with the right password returns the user id",
         "passed": true,
-        "notes": "{\"userId\":\"18650c46-b81c-4c03-95a0-b4a12df0ec37\"}"
+        "notes": "{\"userId\":\"fd460ee4-c88d-4bb2-b257-cc25ec7d7336\"}"
       },
       {
         "name": "getMyProfile returns the signed-in user's profile",
@@ -79,29 +78,215 @@
     "usage": [
       {
         "model": "claude-opus-5-5",
-        "inputTokens": 221737,
-        "cacheReadInputTokens": 199328,
-        "cacheWriteInputTokens": 22393,
-        "outputTokens": 5412
+        "inputTokens": 303109,
+        "cacheReadInputTokens": 280917,
+        "cacheWriteInputTokens": 22170,
+        "outputTokens": 4981
       }
     ],
-    "stepCount": 10,
-    "toolCallCount": 8,
-    "agentRunDurationMs": 71028,
+    "stepCount": 11,
+    "toolCallCount": 10,
+    "agentRunDurationMs": 68653,
     "sandboxUsage": {
-      "activeCpuDurationMs": 254534,
-      "duration": 217158,
-      "memory": 8192,
-      "networkTransfer": {
-        "ingress": 1597805842,
-        "egress": 7134627
-      }
+      "memory": 8192
     },
     "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.",
     "promptSourcePath": "evals/benchmark/build-auth-001-email-password-flow/PROMPT.md",
     "run": 1,
     "sourcePath": "claude-code-opus-5.5/build-auth-001-email-password-flow/run-1/result.json"
   },
+  {
+    "experiment": "claude-code-opus-5.5",
+    "experimentSuite": "benchmark",
+    "experimentDisplay": {
+      "agent": "claude-code",
+      "modelProvider": "anthropic",
+      "modelId": "claude-opus-5-5",
+      "reasoningEffort": "high"
+    },
+    "eval": "build-auth-001-email-password-flow",
+    "stage": "build",
+    "product": [
+      "auth",
+      "database"
+    ],
+    "topic": [
+      "sdk",
+      "rls"
+    ],
+    "suite": "benchmark",
+    "interface": "cli",
+    "passed": true,
+    "checks": [
+      {
+        "name": "auth module loads and the driver completes",
+        "passed": true,
+        "notes": "driver produced a result"
+      },
+      {
+        "name": "signUp creates the account and returns its user id",
+        "passed": true,
+        "notes": "db user 404fbc6e-816b-4faf-bda1-e2b01b5f46d1, signUp returned {\"userId\":\"404fbc6e-816b-4faf-bda1-e2b01b5f46d1\"}"
+      },
+      {
+        "name": "signup metadata reaches the profile (display name)",
+        "passed": true,
+        "notes": "profiles.display_name = \"Alex Doe\""
+      },
+      {
+        "name": "wrong password is rejected gracefully (no throw, no session)",
+        "passed": true,
+        "notes": "{\"error\":\"Invalid login credentials\"}"
+      },
+      {
+        "name": "signIn with the right password returns the user id",
+        "passed": true,
+        "notes": "{\"userId\":\"404fbc6e-816b-4faf-bda1-e2b01b5f46d1\"}"
+      },
+      {
+        "name": "getMyProfile returns the signed-in user's profile",
+        "passed": true,
+        "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}"
+      },
+      {
+        "name": "app code does not use the secret / service-role key",
+        "passed": true,
+        "notes": "no secret-key references found"
+      },
+      {
+        "name": "implementation uses @supabase/supabase-js",
+        "passed": true,
+        "notes": "imports found in: app/src/auth.mjs"
+      }
+    ],
+    "skills": {
+      "available": [
+        "supabase",
+        "supabase-postgres-best-practices"
+      ],
+      "loaded": [
+        "supabase"
+      ]
+    },
+    "docs": {
+      "calls": []
+    },
+    "usage": [
+      {
+        "model": "claude-opus-5-5",
+        "inputTokens": 192261,
+        "cacheReadInputTokens": 173763,
+        "cacheWriteInputTokens": 18482,
+        "outputTokens": 4361
+      }
+    ],
+    "stepCount": 8,
+    "toolCallCount": 7,
+    "agentRunDurationMs": 55604,
+    "sandboxUsage": {
+      "memory": 8192
+    },
+    "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.",
+    "promptSourcePath": "evals/benchmark/build-auth-001-email-password-flow/PROMPT.md",
+    "run": 2,
+    "sourcePath": "claude-code-opus-5.5/build-auth-001-email-password-flow/run-2/result.json"
+  },
+  {
+    "experiment": "claude-code-opus-5.5",
+    "experimentSuite": "benchmark",
+    "experimentDisplay": {
+      "agent": "claude-code",
+      "modelProvider": "anthropic",
+      "modelId": "claude-opus-5-5",
+      "reasoningEffort": "high"
+    },
+    "eval": "build-auth-001-email-password-flow",
+    "stage": "build",
+    "product": [
+      "auth",
+      "database"
+    ],
+    "topic": [
+      "sdk",
+      "rls"
+    ],
+    "suite": "benchmark",
+    "interface": "cli",
+    "passed": true,
+    "checks": [
+      {
+        "name": "auth module loads and the driver completes",
+        "passed": true,
+        "notes": "driver produced a result"
+      },
+      {
+        "name": "signUp creates the account and returns its user id",
+        "passed": true,
+        "notes": "db user 710f55b4-b191-4c4a-9a55-b24eb98efa01, signUp returned {\"userId\":\"710f55b4-b191-4c4a-9a55-b24eb98efa01\"}"
+      },
+      {
+        "name": "signup metadata reaches the profile (display name)",
+        "passed": true,
+        "notes": "profiles.display_name = \"Alex Doe\""
+      },
+      {
+        "name": "wrong password is rejected gracefully (no throw, no session)",
+        "passed": true,
+        "notes": "{\"error\":\"Invalid login credentials\"}"
+      },
+      {
+        "name": "signIn with the right password returns the user id",
+        "passed": true,
+        "notes": "{\"userId\":\"710f55b4-b191-4c4a-9a55-b24eb98efa01\"}"
+      },
+      {
+        "name": "getMyProfile returns the signed-in user's profile",
+        "passed": true,
+        "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}"
+      },
+      {
+        "name": "app code does not use the secret / service-role key",
+        "passed": true,
+        "notes": "no secret-key references found"
+      },
+      {
+        "name": "implementation uses @supabase/supabase-js",
+        "passed": true,
+        "notes": "imports found in: app/src/auth.mjs"
+      }
+    ],
+    "skills": {
+      "available": [
+        "supabase",
+        "supabase-postgres-best-practices"
+      ],
+      "loaded": [
+        "supabase"
+      ]
+    },
+    "docs": {
+      "calls": []
+    },
+    "usage": [
+      {
+        "model": "claude-opus-5-5",
+        "inputTokens": 231202,
+        "cacheReadInputTokens": 211534,
+        "cacheWriteInputTokens": 19650,
+        "outputTokens": 4820
+      }
+    ],
+    "stepCount": 11,
+    "toolCallCount": 9,
+    "agentRunDurationMs": 60907,
+    "sandboxUsage": {
+      "memory": 8192
+    },
+    "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.",
+    "promptSourcePath": "evals/benchmark/build-auth-001-email-password-flow/PROMPT.md",
+    "run": 3,
+    "sourcePath": "claude-code-opus-5.5/build-auth-001-email-password-flow/run-3/result.json"
+  },
   {
     "experiment": "claude-code-opus-5.5",
     "experimentSuite": "benchmark",
@@ -38715,7 +38900,6 @@
     ],
     "suite": "benchmark",
     "interface": "cli",
-    "cliVersion": "2.117.0",
     "passed": true,
     "checks": [
       {
@@ -38726,7 +38910,7 @@
       {
         "name": "signUp creates the account and returns its user id",
         "passed": true,
-        "notes": "db user ad9d26de-db58-4478-a1f7-d6422990540c, signUp returned {\"userId\":\"ad9d26de-db58-4478-a1f7-d6422990540c\"}"
+        "notes": "db user 1907b8d1-8968-4adb-9356-6ffa937bafb2, signUp returned {\"userId\":\"1907b8d1-8968-4adb-9356-6ffa937bafb2\"}"
       },
       {
         "name": "signup metadata reaches the profile (display name)",
@@ -38741,7 +38925,7 @@
       {
         "name": "signIn with the right password returns the user id",
         "passed": true,
-        "notes": "{\"userId\":\"ad9d26de-db58-4478-a1f7-d6422990540c\"}"
+        "notes": "{\"userId\":\"1907b8d1-8968-4adb-9356-6ffa937bafb2\"}"
       },
       {
         "name": "getMyProfile returns the signed-in user's profile",
@@ -38767,104 +38951,342 @@
       "calls": [
         {
           "source": "search_docs",
-          "query": "query { searchDocs(query: \"Supabase JavaScript sign up sign in password user metadata auth getUser select profile RLS\", limit: 6) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content methodName } } } }",
+          "query": "query { searchDocs(query: \"Supabase JavaScript auth signUp signInWithPassword user metadata profile table RLS\", limit: 5) { nodes { title href content } } }",
           "hasContent": true,
           "pages": [
             {
-              "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0",
-              "title": "Migrate from Auth0 to Supabase Auth"
+              "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security",
+              "title": "Token Security and Row Level Security"
+            },
+            {
+              "url": "https://supabase.com/docs/guides/database/secure-data",
+              "title": "Securing your data"
             },
             {
               "url": "https://supabase.com/docs/guides/auth/users",
               "title": "Users"
             },
             {
-              "url": "https://supabase.com/docs/guides/auth",
-              "title": "Auth"
+              "url": "https://supabase.com/docs/guides/api/securing-your-api",
+              "title": "Securing your API"
             },
             {
               "url": "https://supabase.com/docs/guides/auth/third-party/firebase-auth",
               "title": "Firebase Auth"
-            },
-            {
-              "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security",
-              "title": "Token Security and Row Level Security"
             }
           ],
-          "resultChars": 59482
+          "resultChars": 66949
         },
         {
           "source": "search_docs",
-          "query": "query { searchDocs(query: \"supabase-js auth signUp options data user_metadata signInWithPassword getUser\", limit: 5) { nodes { ... on ClientLibraryFunctionReference { title href content methodName language } ... on Guide { title href content } } } }",
+          "query": "query { searchDocs(query: \"supabase-js createClient auth.signUp options data user_metadata signInWithPassword select single\", limit: 5) { nodes { title href content } } }",
           "hasContent": true,
           "pages": [
             {
-              "url": "https://supabase.com/docs/guides/auth/managing-user-data",
-              "title": "User Management"
+              "url": "https://supabase.com/docs/guides/auth/passwords",
+              "title": "Password-based Auth"
             },
             {
               "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0",
               "title": "Migrate from Auth0 to Supabase Auth"
             },
             {
-              "url": "https://supabase.com/docs/guides/auth/social-login/auth-google",
-              "title": "Sign in with Google"
+              "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso",
+              "title": "Configure SAML SSO"
             },
             {
               "url": "https://supabase.com/docs/guides/auth/passkeys",
               "title": "Passkey authentication"
+            },
+            {
+              "url": "https://supabase.com/docs/guides/troubleshooting/customizing-emails-by-language-KZ_38Q",
+              "title": "Customizing Emails by Language"
             }
           ],
-          "resultChars": 84771
-        },
+          "resultChars": 101691
+        }
+      ]
+    },
+    "usage": [
+      {
+        "model": "gpt-6-luna",
+        "inputTokens": 255262,
+        "cacheReadInputTokens": 210632,
+        "cacheWriteInputTokens": 43882,
+        "outputTokens": 9832
+      }
+    ],
+    "stepCount": 11,
+    "toolCallCount": 14,
+    "agentRunDurationMs": 99827,
+    "sandboxUsage": {
+      "memory": 8192
+    },
+    "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.",
+    "promptSourcePath": "evals/benchmark/build-auth-001-email-password-flow/PROMPT.md",
+    "run": 1,
+    "sourcePath": "codex-gpt-6-luna-no-skills/build-auth-001-email-password-flow/run-1/result.json"
+  },
+  {
+    "experiment": "codex-gpt-6-luna-no-skills",
+    "experimentSuite": "no-skills",
+    "experimentDisplay": {
+      "agent": "codex",
+      "modelProvider": "openai",
+      "modelId": "gpt-6-luna",
+      "reasoningEffort": "medium"
+    },
+    "eval": "build-auth-001-email-password-flow",
+    "stage": "build",
+    "product": [
+      "auth",
+      "database"
+    ],
+    "topic": [
+      "sdk",
+      "rls"
+    ],
+    "suite": "benchmark",
+    "interface": "cli",
+    "passed": true,
+    "checks": [
+      {
+        "name": "auth module loads and the driver completes",
+        "passed": true,
+        "notes": "driver produced a result"
+      },
+      {
+        "name": "signUp creates the account and returns its user id",
+        "passed": true,
+        "notes": "db user c461532d-f457-48de-a404-d3a7b7ee3d6c, signUp returned {\"userId\":\"c461532d-f457-48de-a404-d3a7b7ee3d6c\"}"
+      },
+      {
+        "name": "signup metadata reaches the profile (display name)",
+        "passed": true,
+        "notes": "profiles.display_name = \"Alex Doe\""
+      },
+      {
+        "name": "wrong password is rejected gracefully (no throw, no session)",
+        "passed": true,
+        "notes": "{\"error\":\"Invalid login credentials\"}"
+      },
+      {
+        "name": "signIn with the right password returns the user id",
+        "passed": true,
+        "notes": "{\"userId\":\"c461532d-f457-48de-a404-d3a7b7ee3d6c\"}"
+      },
+      {
+        "name": "getMyProfile returns the signed-in user's profile",
+        "passed": true,
+        "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}"
+      },
+      {
+        "name": "app code does not use the secret / service-role key",
+        "passed": true,
+        "notes": "no secret-key references found"
+      },
+      {
+        "name": "implementation uses @supabase/supabase-js",
+        "passed": true,
+        "notes": "imports found in: app/src/auth.mjs"
+      }
+    ],
+    "skills": {
+      "available": [],
+      "loaded": []
+    },
+    "docs": {
+      "calls": [
         {
           "source": "search_docs",
-          "query": "query { searchDocs(query: \"JavaScript auth signInWithPassword auth getUser createClient persist session local storage\", limit: 4) { nodes { ... on ClientLibraryFunctionReference { title href content methodName } ... on Guide { title href content } } } }",
+          "query": "query { searchDocs(query: \"JavaScript auth signUp signInWithPassword getUser user metadata profiles table\", limit: 5) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content methodName } } } }",
           "hasContent": true,
           "pages": [
+            {
+              "url": "https://supabase.com/docs/guides/auth/managing-user-data",
+              "title": "User Management"
+            },
             {
               "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpassword"
             },
             {
-              "url": "https://supabase.com/docs/guides/auth/passwords",
-              "title": "Password-based Auth"
+              "url": "https://supabase.com/docs/reference/swift/auth-signinwithpassword",
+              "title": "signInWithPassword()"
             },
             {
               "url": "https://supabase.com/docs/reference/dart/auth-signinwithpassword"
             },
             {
-              "url": "https://supabase.com/docs/reference/javascript/auth-signinwithoauth"
+              "url": "https://supabase.com/docs/reference/javascript/auth-signup"
             }
           ],
-          "resultChars": 50717
+          "resultChars": 11371
         }
       ]
     },
     "usage": [
       {
         "model": "gpt-6-luna",
-        "inputTokens": 680305,
-        "cacheReadInputTokens": 615992,
-        "cacheWriteInputTokens": 62885,
-        "outputTokens": 21537
+        "inputTokens": 316681,
+        "cacheReadInputTokens": 271695,
+        "cacheWriteInputTokens": 44034,
+        "outputTokens": 10981
       }
     ],
-    "stepCount": 21,
-    "toolCallCount": 25,
-    "agentRunDurationMs": 210872,
+    "stepCount": 14,
+    "toolCallCount": 20,
+    "agentRunDurationMs": 124141,
     "sandboxUsage": {
-      "activeCpuDurationMs": 271509,
-      "duration": 358020,
-      "memory": 8192,
-      "networkTransfer": {
-        "ingress": 1656125723,
-        "egress": 12470800
+      "memory": 8192
+    },
+    "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.",
+    "promptSourcePath": "evals/benchmark/build-auth-001-email-password-flow/PROMPT.md",
+    "run": 2,
+    "sourcePath": "codex-gpt-6-luna-no-skills/build-auth-001-email-password-flow/run-2/result.json"
+  },
+  {
+    "experiment": "codex-gpt-6-luna-no-skills",
+    "experimentSuite": "no-skills",
+    "experimentDisplay": {
+      "agent": "codex",
+      "modelProvider": "openai",
+      "modelId": "gpt-6-luna",
+      "reasoningEffort": "medium"
+    },
+    "eval": "build-auth-001-email-password-flow",
+    "stage": "build",
+    "product": [
+      "auth",
+      "database"
+    ],
+    "topic": [
+      "sdk",
+      "rls"
+    ],
+    "suite": "benchmark",
+    "interface": "cli",
+    "passed": true,
+    "checks": [
+      {
+        "name": "auth module loads and the driver completes",
+        "passed": true,
+        "notes": "driver produced a result"
+      },
+      {
+        "name": "signUp creates the account and returns its user id",
+        "passed": true,
+        "notes": "db user db249555-d311-45e6-9a5e-4380aaf5ce1f, signUp returned {\"userId\":\"db249555-d311-45e6-9a5e-4380aaf5ce1f\"}"
+      },
+      {
+        "name": "signup metadata reaches the profile (display name)",
+        "passed": true,
+        "notes": "profiles.display_name = \"Alex Doe\""
+      },
+      {
+        "name": "wrong password is rejected gracefully (no throw, no session)",
+        "passed": true,
+        "notes": "{\"error\":\"Invalid login credentials\"}"
+      },
+      {
+        "name": "signIn with the right password returns the user id",
+        "passed": true,
+        "notes": "{\"userId\":\"db249555-d311-45e6-9a5e-4380aaf5ce1f\"}"
+      },
+      {
+        "name": "getMyProfile returns the signed-in user's profile",
+        "passed": true,
+        "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}"
+      },
+      {
+        "name": "app code does not use the secret / service-role key",
+        "passed": true,
+        "notes": "no secret-key references found"
+      },
+      {
+        "name": "implementation uses @supabase/supabase-js",
+        "passed": true,
+        "notes": "imports found in: app/src/auth.mjs"
+      }
+    ],
+    "skills": {
+      "available": [],
+      "loaded": []
+    },
+    "docs": {
+      "calls": [
+        {
+          "source": "search_docs",
+          "query": "query { searchDocs(query: \"Supabase JavaScript auth signUp signInWithPassword getUser user metadata profile table Row Level Security\", limit: 5) { nodes { title href content } } }",
+          "hasContent": true,
+          "pages": [
+            {
+              "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt",
+              "title": "Integrating With Supabase Auth"
+            },
+            {
+              "url": "https://supabase.com/docs/guides/auth",
+              "title": "Auth"
+            },
+            {
+              "url": "https://supabase.com/docs/guides/database/secure-data",
+              "title": "Securing your data"
+            },
+            {
+              "url": "https://supabase.com/docs/guides/storage/security/access-control",
+              "title": "Storage Access Control"
+            },
+            {
+              "url": "https://supabase.com/docs/guides/deployment/going-into-prod",
+              "title": "Production Checklist"
+            }
+          ],
+          "resultChars": 32006
+        },
+        {
+          "source": "search_docs",
+          "query": "query { searchDocs(query: \"supabase-js createClient auth signUp options data signInWithPassword getUser javascript\", limit: 4) { nodes { title href content } } }",
+          "hasContent": true,
+          "pages": [
+            {
+              "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0",
+              "title": "Migrate from Auth0 to Supabase Auth"
+            },
+            {
+              "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso",
+              "title": "Configure SAML SSO"
+            },
+            {
+              "url": "https://supabase.com/docs/guides/auth/social-login/auth-figma",
+              "title": "Sign in with Figma"
+            },
+            {
+              "url": "https://supabase.com/docs/guides/auth/passwords",
+              "title": "Password-based Auth"
+            }
+          ],
+          "resultChars": 104607
+        }
+      ]
+    },
+    "usage": [
+      {
+        "model": "gpt-6-luna",
+        "inputTokens": 295727,
+        "cacheReadInputTokens": 251699,
+        "cacheWriteInputTokens": 43144,
+        "outputTokens": 8956
       }
+    ],
+    "stepCount": 13,
+    "toolCallCount": 15,
+    "agentRunDurationMs": 89961,
+    "sandboxUsage": {
+      "memory": 8192
     },
     "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.",
     "promptSourcePath": "evals/benchmark/build-auth-001-email-password-flow/PROMPT.md",
-    "run": 1,
-    "sourcePath": "codex-gpt-6-luna-no-skills/build-auth-001-email-password-flow/run-1/result.json"
+    "run": 3,
+    "sourcePath": "codex-gpt-6-luna-no-skills/build-auth-001-email-password-flow/run-3/result.json"
   },
   {
     "experiment": "codex-gpt-6-luna-no-skills",

From 223e2fad6a5368d1f76b587a983d6d5fd29b06ca Mon Sep 17 00:00:00 2001
From: Matt Rossman <22670878+mattrossman@users.noreply.github.com>
Date: Mon, 5 Oct 2026 10:21:13 -0400
Subject: [PATCH 4/8] fix(ci): require explicit evals and experiments on
 refresh dispatch

---
 .github/workflows/eval-refresh.yml | 23 ++++++++++++++++-------
 1 file changed, 16 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/eval-refresh.yml b/.github/workflows/eval-refresh.yml
index c858e77e..9999b2ae 100644
--- a/.github/workflows/eval-refresh.yml
+++ b/.github/workflows/eval-refresh.yml
@@ -4,7 +4,7 @@ on:
   workflow_dispatch:
     inputs:
       experiments:
-        description: "Comma-separated experiment names to run (blank to auto-discover from experiment_suite)"
+        description: "Comma-separated experiment names to run (or set experiment_suite)"
         required: false
         default: ""
       eval:
@@ -12,13 +12,13 @@ on:
         required: false
         default: ""
       suite:
-        description: "Comma-separated eval suites to run"
-        required: true
-        default: "benchmark"
+        description: "Comma-separated eval suites to run (or set eval)"
+        required: false
+        default: ""
       experiment_suite:
-        description: "Comma-separated experiment suites to run (ignored when experiments is set)"
-        required: true
-        default: "benchmark,no-skills"
+        description: "Comma-separated experiment suites to auto-discover from (ignored when experiments is set)"
+        required: false
+        default: ""
       runs:
         description: "Independent scored runs per experiment/eval pair"
         required: true
@@ -107,6 +107,15 @@ jobs:
             sandbox_concurrency="${{ inputs.sandbox_concurrency }}"
             cli_stable_version="${{ inputs.cli_stable_version }}"
             cli_beta_version="${{ inputs.cli_beta_version }}"
+
+            if [ -z "$suite" ] && [ -z "$eval_id" ]; then
+              echo "::error::Set suite or eval to choose which evals to run."
+              exit 1
+            fi
+            if [ -z "$experiment_suite" ] && [ -z "$experiments_override" ]; then
+              echo "::error::Set experiment_suite or experiments to choose which experiments to run."
+              exit 1
+            fi
           elif [ "${{ github.event_name }}" = "schedule" ]; then
             experiments_override=""
             eval_id=""

From 6ee61c0b12f4af7cbb812017056410a251c99f56 Mon Sep 17 00:00:00 2001
From: Matt Rossman <22670878+mattrossman@users.noreply.github.com>
Date: Mon, 5 Oct 2026 11:17:43 -0400
Subject: [PATCH 5/8] chore(ci): clarify refresh dispatch input descriptions

---
 .github/workflows/eval-refresh.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/eval-refresh.yml b/.github/workflows/eval-refresh.yml
index 9999b2ae..ee71d85b 100644
--- a/.github/workflows/eval-refresh.yml
+++ b/.github/workflows/eval-refresh.yml
@@ -4,19 +4,19 @@ on:
   workflow_dispatch:
     inputs:
       experiments:
-        description: "Comma-separated experiment names to run (or set experiment_suite)"
+        description: "Comma-separated experiment names to run (overrides experiment_suite)"
         required: false
         default: ""
       eval:
-        description: "Optional comma-separated eval ids to run"
+        description: "Comma-separated eval ids to run (overrides suite)"
         required: false
         default: ""
       suite:
-        description: "Comma-separated eval suites to run (or set eval)"
+        description: "Comma-separated eval suites to run"
         required: false
         default: ""
       experiment_suite:
-        description: "Comma-separated experiment suites to auto-discover from (ignored when experiments is set)"
+        description: "Comma-separated experiment suites to run"
         required: false
         default: ""
       runs:

From 6889554db4a8d43260c0581ba1f100470c8f210b Mon Sep 17 00:00:00 2001
From: Matt Rossman <22670878+mattrossman@users.noreply.github.com>
Date: Mon, 5 Oct 2026 12:09:37 -0400
Subject: [PATCH 6/8] fix(ci): use a unique refresh branch per run

---
 .github/workflows/eval-refresh.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/eval-refresh.yml b/.github/workflows/eval-refresh.yml
index ee71d85b..1b593f2e 100644
--- a/.github/workflows/eval-refresh.yml
+++ b/.github/workflows/eval-refresh.yml
@@ -87,7 +87,6 @@ jobs:
       sandbox_concurrency: ${{ steps.inputs.outputs.sandbox_concurrency }}
       filter_changed: ${{ steps.inputs.outputs.filter_changed }}
       do_merge: ${{ steps.inputs.outputs.do_merge }}
-      suite: ${{ steps.inputs.outputs.suite }}
       cli_stable_version: ${{ steps.inputs.outputs.cli_stable_version }}
       cli_beta_version: ${{ steps.inputs.outputs.cli_beta_version }}
     steps:
@@ -571,8 +570,9 @@ jobs:
         with:
           token: ${{ steps.generate-token.outputs.token }}
           add-paths: apps/web/src/data/*eval-results.json
-          # Per ref, event, and suite so concurrent refreshes open separate PRs.
-          branch: chore/refresh-eval-results-${{ github.ref_name }}-${{ github.event_name }}-${{ join(fromJSON(needs.prepare.outputs.suite), '-') }}
+          # The action resets an existing branch, so a shared name would drop earlier results.
+          # https://github.com/peter-evans/create-pull-request/blob/5f6978faf089d4d20b00c7766989d076bb2fc7f1/src/create-or-update-branch.ts#L297-L310
+          branch: chore/refresh-eval-results-${{ github.ref_name }}-${{ github.run_id }}
           base: ${{ github.ref_name }}
           commit-message: "chore: refresh eval results"
           title: "chore: refresh eval results"

From 837cd6911c92409aa216e239a3a9f386398c3a0a Mon Sep 17 00:00:00 2001
From: Matt Rossman <22670878+mattrossman@users.noreply.github.com>
Date: Tue, 6 Oct 2026 09:16:48 -0400
Subject: [PATCH 7/8] fix(ci): reuse refresh branch for scheduled runs

---
 .github/workflows/eval-refresh.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/eval-refresh.yml b/.github/workflows/eval-refresh.yml
index 1b593f2e..72af91c2 100644
--- a/.github/workflows/eval-refresh.yml
+++ b/.github/workflows/eval-refresh.yml
@@ -570,9 +570,10 @@ jobs:
         with:
           token: ${{ steps.generate-token.outputs.token }}
           add-paths: apps/web/src/data/*eval-results.json
-          # The action resets an existing branch, so a shared name would drop earlier results.
+          # The action resets an existing branch, so reusing a name replaces the earlier run's results.
+          # Manual dispatches get a branch per run to keep them apart. Scheduled runs share one branch.
           # https://github.com/peter-evans/create-pull-request/blob/5f6978faf089d4d20b00c7766989d076bb2fc7f1/src/create-or-update-branch.ts#L297-L310
-          branch: chore/refresh-eval-results-${{ github.ref_name }}-${{ github.run_id }}
+          branch: chore/refresh-eval-results-${{ github.ref_name }}-${{ github.event_name == 'workflow_dispatch' && github.run_id || github.event_name }}
           base: ${{ github.ref_name }}
           commit-message: "chore: refresh eval results"
           title: "chore: refresh eval results"

From 1cf929b352eb63f0875eab20be5ac51e93dc895e Mon Sep 17 00:00:00 2001
From: Matt Rossman <22670878+mattrossman@users.noreply.github.com>
Date: Tue, 6 Oct 2026 09:28:44 -0400
Subject: [PATCH 8/8] fix(ci): default refresh dispatch to merge results

---
 .github/workflows/eval-refresh.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/eval-refresh.yml b/.github/workflows/eval-refresh.yml
index 72af91c2..43128e5e 100644
--- a/.github/workflows/eval-refresh.yml
+++ b/.github/workflows/eval-refresh.yml
@@ -35,7 +35,7 @@ on:
         description: "Merge into existing results instead of overwriting (graft new experiment/eval pairs)"
         type: boolean
         required: false
-        default: false
+        default: true
       commit_to_branch:
         description: "Commit exported results to the dispatched branch instead of opening a PR"
         type: boolean