From c1f2203a17f8a2c9f39a7fc274db7426dcd64502 Mon Sep 17 00:00:00 2001 From: Matt Rossman <22670878+mattrossman@users.noreply.github.com> Date: Tue, 6 Oct 2026 17:14:20 -0400 Subject: [PATCH 01/33] fix(core): expect id on Codex tool calls in parser test --- packages/core/src/agents/codex/parser.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/core/src/agents/codex/parser.test.ts b/packages/core/src/agents/codex/parser.test.ts index 416098b0..f1e54752 100644 --- a/packages/core/src/agents/codex/parser.test.ts +++ b/packages/core/src/agents/codex/parser.test.ts @@ -232,6 +232,7 @@ describe('codexParser', () => { const adapted = adaptTranscript(codexParser.parseTranscript(stream).events); expect(adapted.transcript[0]).toEqual({ type: 'tool_call', + id: 'item_7', name: 'execute_sql', input: args, output: toolResult, From 323cada8b494feed43d0316bafd7946d54d2311e Mon Sep 17 00:00:00 2001 From: Matt Rossman <22670878+mattrossman@users.noreply.github.com> Date: Tue, 6 Oct 2026 17:14:20 -0400 Subject: [PATCH 02/33] feat(ci): run tests on pull requests --- .github/workflows/eval-refresh.yml | 17 ++++----- .github/workflows/test.yml | 55 ++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 10 deletions(-) create mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/eval-refresh.yml b/.github/workflows/eval-refresh.yml index 43128e5e..07177104 100644 --- a/.github/workflows/eval-refresh.yml +++ b/.github/workflows/eval-refresh.yml @@ -62,7 +62,8 @@ permissions: actions: read concurrency: - group: eval-refresh-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.ref }} + # The app's own results push gets a unique group so it can't cancel the run that pushed it. + group: eval-refresh-${{ github.event.sender.login == 'supabase-evals-releaser[bot]' && github.run_id || github.event_name == 'pull_request' && github.event.pull_request.number || github.ref }} # Supersede an in-flight run when a new commit is pushed to the same PR, but # never cancel a workflow_dispatch run (those open the refresh PR). cancel-in-progress: ${{ github.event_name == 'pull_request' }} @@ -73,6 +74,7 @@ jobs: github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' || (github.event_name == 'pull_request' && + github.event.sender.login != 'supabase-evals-releaser[bot]' && (contains(github.event.pull_request.labels.*.name, 'run-evals') || contains(github.event.pull_request.labels.*.name, 'run-evals-changed')) && github.event.pull_request.head.repo.full_name == github.repository && @@ -375,12 +377,8 @@ jobs: steps: - name: Generate GitHub App token id: generate-token - # A GitHub App token is needed so the push below can trigger the - # gh-pages workflow: GITHUB_TOKEN pushes don't trigger other workflows. + # The App token lets the pushes below trigger gh-pages and PR checks. # https://docs.github.com/en/actions/how-tos/write-workflows/choose-when-workflows-run/trigger-a-workflow#triggering-a-workflow-from-a-workflow - if: >- - github.event_name == 'schedule' || - (github.event_name == 'workflow_dispatch' && !inputs.commit_to_branch) uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 with: client-id: ${{ secrets.GH_APP_ID }} @@ -390,7 +388,7 @@ jobs: uses: actions/checkout@9f698171ed81b15d1823a05fc7211befd50c8ae0 # v6.0.3 with: ref: ${{ github.event_name == 'pull_request' && github.head_ref || github.ref }} - token: ${{ steps.generate-token.outputs.token || github.token }} + token: ${{ steps.generate-token.outputs.token }} - name: Install pnpm uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 @@ -556,9 +554,8 @@ jobs: exit 0 fi - # [skip ci] stops the bot push from re-triggering PR workflows (they'd need maintainer approval and loop the refresh): - # https://docs.github.com/en/actions/managing-workflow-runs-and-deployments/managing-workflow-runs/skipping-workflow-runs - git commit -m "chore: refresh eval results [skip ci]" + # Required PR checks run on this commit. prepare skips the app's pushes so the refresh doesn't loop. + git commit -m "chore: refresh eval results" git push - name: Create results pull request diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..de7de978 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,55 @@ +name: Tests + +on: + pull_request: + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-26.04 + steps: + - name: Checkout + uses: actions/checkout@9f698171ed81b15d1823a05fc7211befd50c8ae0 # v6.0.3 + with: + persist-credentials: false + + - name: Install pnpm + uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 + + - name: Setup Node.js + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version-file: .node-version + cache: pnpm + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Run checks + run: pnpm check + + # Own job since it's the slowest suite. Every test boots fresh PGlite databases. + test-platform-lite: + runs-on: ubuntu-26.04 + steps: + - name: Checkout + uses: actions/checkout@9f698171ed81b15d1823a05fc7211befd50c8ae0 # v6.0.3 + with: + persist-credentials: false + + - name: Install pnpm + uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 + + - name: Setup Node.js + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version-file: .node-version + cache: pnpm + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Run platform-lite tests + run: pnpm --filter @supabase-evals/platform-lite test From 9c94bd1fe4e4d9ecef0fc3169297f40947cf0e80 Mon Sep 17 00:00:00 2001 From: Matt Rossman <22670878+mattrossman@users.noreply.github.com> Date: Tue, 6 Oct 2026 17:14:38 -0400 Subject: [PATCH 03/33] chore: temporary eval change for refresh pilot --- .../resolve-reliability-001-unhealthy-project-recovery/EVAL.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/evals/regression/resolve-reliability-001-unhealthy-project-recovery/EVAL.ts b/evals/regression/resolve-reliability-001-unhealthy-project-recovery/EVAL.ts index d1382da7..d9c492ac 100644 --- a/evals/regression/resolve-reliability-001-unhealthy-project-recovery/EVAL.ts +++ b/evals/regression/resolve-reliability-001-unhealthy-project-recovery/EVAL.ts @@ -35,3 +35,5 @@ const scorer: ToolScorer = async (ctx) => { }; export default scorer; + +// Temporary change to exercise the refresh results commit. From 03ab19e1bf3dc269f17087d76dd12c8cd5ae630f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 6 Oct 2026 21:18:50 +0000 Subject: [PATCH 04/33] chore: refresh eval results --- .../web/src/data/regression-eval-results.json | 375 +++++++++--------- 1 file changed, 182 insertions(+), 193 deletions(-) diff --git a/apps/web/src/data/regression-eval-results.json b/apps/web/src/data/regression-eval-results.json index f9d78302..2f0fe549 100644 --- a/apps/web/src/data/regression-eval-results.json +++ b/apps/web/src/data/regression-eval-results.json @@ -5132,7 +5132,7 @@ { "name": "answered unhealthy project recovery question safely", "passed": true, - "judgeNotes": "Recommends a restart as the first recovery step, distinguishes it from backup restoration, and warns against pause/resume for an unhealthy project. It also points to incident checks and resource-pressure investigation." + "judgeNotes": "Recommends a database restart as the concrete first step and rejects pause/restore as an unhealthy-project workaround. It could more clearly explain that backup restore is data recovery and advise contacting support if the project remains unhealthy." } ], "skills": { @@ -5149,27 +5149,23 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase project restart pause restore project dashboard unavailable troubleshooting monitoring debugging\", limit: 5) { nodes { title href content } }", + "query": "query { searchDocs(query: \"Monitoring and Debugging Supabase project unhealthy project dashboard\", limit: 5) { nodes { title href content } }", "hasContent": true, "pages": [] }, { "source": "search_docs", - "query": "query { searchDocs(query: \"project unhealthy restart pause restore project recovery dashboard unhealthy\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"project unhealthy restart pause restore project dashboard project recovery\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services", - "title": "Project Status reports unhealthy services" + "url": "https://supabase.com/docs/guides/platform/free-project-pausing", + "title": "Project Pausing" }, { "url": "https://supabase.com/docs/guides/platform/delete-project", "title": "Deleting Your Project" }, - { - "url": "https://supabase.com/docs/guides/platform/free-project-pausing", - "title": "Project Pausing" - }, { "url": "https://supabase.com/docs/guides/platform/upgrading", "title": "Upgrading" @@ -5177,46 +5173,79 @@ { "url": "https://supabase.com/docs/guides/platform/backups", "title": "Database Backups" + }, + { + "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", + "title": "Restoring a downloaded backup locally" } ], - "resultChars": 32572 + "resultChars": 32742 }, { - "source": "web_search", - "query": "site:supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services restart database unhealthy services", + "source": "search_docs", + "query": "query { searchDocs(query: \"Monitoring and Debugging Supabase projects unhealthy status\", limit: 5) { nodes { title href content } }", + "hasContent": true, "pages": [] }, { - "source": "web_search", - "query": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services", + "source": "search_docs", + "query": "query { searchDocs(query: \"restart project dashboard unhealthy restart project Supabase\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services" + "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", + "title": "Restoring a downloaded backup locally" + }, + { + "url": "https://supabase.com/docs/guides/ai/examples/nextjs-vector-search", + "title": "Vector search with Next.js and OpenAI" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/remove-superuser-access", + "title": "Remove Superuser Access from Studio" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-expo-react-native", + "title": "Build a User Management App with Expo React Native" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-ionic-vue", + "title": "Build a User Management App with Ionic Vue" } - ] + ], + "resultChars": 89339 + }, + { + "source": "web_search", + "query": "site:supabase.com/docs project unhealthy restart project Supabase restart", + "pages": [] + }, + { + "source": "web_search", + "query": "Supabase status current incidents status.supabase.com October 6 2026", + "pages": [] } ] }, "usage": [ { "model": "gpt-6-luna", - "inputTokens": 69751, - "cacheReadInputTokens": 16870, - "cacheWriteInputTokens": 32169, - "outputTokens": 2232 + "inputTokens": 87152, + "cacheReadInputTokens": 36795, + "cacheWriteInputTokens": 35584, + "outputTokens": 1566 } ], - "stepCount": 3, - "toolCallCount": 7, - "agentRunDurationMs": 44205, + "stepCount": 4, + "toolCallCount": 9, + "agentRunDurationMs": 29124, "sandboxUsage": { - "activeCpuDurationMs": 182017, - "duration": 172392, + "activeCpuDurationMs": 130367, + "duration": 130067, "memory": 8192, "networkTransfer": { - "ingress": 762143255, - "egress": 1795863 + "ingress": 764300671, + "egress": 3850265 } }, "prompt": "My Supabase dashboard says my project is unhealthy, and the dashboard is unusable.\n\nWould restart or pause/restore be better?", @@ -5248,7 +5277,7 @@ { "name": "answered unhealthy project recovery question safely", "passed": true, - "judgeNotes": "Recommends restarting rather than pause/restore, distinguishes the operations, cites Supabase’s unhealthy-services guidance, and directs the user to support if the controls are unavailable. It could more clearly describe restore as data recovery and mention checking logs and resource pressure." + "judgeNotes": "Recommends restarting the database first, rejects pause/restore as a troubleshooting workaround, distinguishes it from a routine restart, cites Supabase’s unhealthy-services guidance, and advises contacting support if restarting fails." } ], "skills": { @@ -5264,47 +5293,73 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase project unhealthy restart pause restore project dashboard recovery\", limit: 8) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"restore paused project Supabase dashboard restart project\", limit: 5) { nodes { title href content } }", + "hasContent": true, + "pages": [] + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Monitoring and Debugging project unhealthy restart project pause restore project\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/platform/free-project-pausing", - "title": "Project Pausing" - }, { "url": "https://supabase.com/docs/guides/platform/upgrading", "title": "Upgrading" }, { - "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", - "title": "Restoring a downloaded backup locally" + "url": "https://supabase.com/docs/guides/platform/free-project-pausing", + "title": "Project Pausing" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/restore-project-after-90-days-pause", - "title": "How To Restore a Project Paused for More Than 1 Year" + "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", + "title": "Restoring a downloaded backup locally" }, { "url": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services", "title": "Project Status reports unhealthy services" }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/pausing-pro-projects-vNL-2a", - "title": "Pausing Pro-Projects" - }, { "url": "https://supabase.com/docs/guides/platform/delete-project", "title": "Deleting Your Project" + } + ], + "resultChars": 24720 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"project unhealthy status health check logs restart database Supabase\", limit: 3) { nodes { title href content } }", + "hasContent": true, + "pages": [] + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"pause restore project database backup limitations downtime Supabase\", limit: 3) { nodes { title href content } }", + "hasContent": true, + "pages": [] + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Monitoring and Debugging unhealthy project restart database health checks\", limit: 3) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services", + "title": "Project Status reports unhealthy services" + }, + { + "url": "https://supabase.com/docs/guides/observability/detecting", + "title": "Detection checks" }, { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres", - "title": "Migrate from Postgres to Supabase" + "url": "https://supabase.com/docs/guides/troubleshooting/failed-to-run-sql-query-connection-terminated-due-to-connection-timeout", + "title": "Error: Failed to run sql query: Connection terminated due to connection timeout" } ], - "resultChars": 51101 + "resultChars": 14753 }, { "source": "web_search", - "query": "site:supabase.com/docs restart project Supabase dashboard restart project unhealthy", + "query": "site:supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services restart database unhealthy", "pages": [] } ] @@ -5312,22 +5367,22 @@ "usage": [ { "model": "gpt-6-luna", - "inputTokens": 57617, - "cacheReadInputTokens": 16916, - "cacheWriteInputTokens": 32169, - "outputTokens": 1268 + "inputTokens": 77010, + "cacheReadInputTokens": 36918, + "cacheWriteInputTokens": 35659, + "outputTokens": 1703 } ], - "stepCount": 3, - "toolCallCount": 3, - "agentRunDurationMs": 29120, + "stepCount": 4, + "toolCallCount": 7, + "agentRunDurationMs": 30142, "sandboxUsage": { - "activeCpuDurationMs": 185218, - "duration": 160268, + "activeCpuDurationMs": 129484, + "duration": 128214, "memory": 8192, "networkTransfer": { - "ingress": 762731208, - "egress": 1934399 + "ingress": 764832046, + "egress": 3719151 } }, "prompt": "My Supabase dashboard says my project is unhealthy, and the dashboard is unusable.\n\nWould restart or pause/restore be better?", @@ -5359,7 +5414,7 @@ { "name": "answered unhealthy project recovery question safely", "passed": true, - "judgeNotes": "Recommends restart as the first recovery step, warns against pause/restore as a shortcut, and points to workload checks, troubleshooting guidance, and support. It could more clearly explain that backup restore is separate from restart." + "judgeNotes": "Rejects pause/restore as a health reset, distinguishes backup restore from a database restart, recommends a conditional restart for unhealthy services, and advises contacting support if access remains unavailable." } ], "skills": { @@ -5368,130 +5423,74 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase", - "supabase-postgres-best-practices" + "supabase" ] }, "docs": { "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"restart project pause restore unhealthy project dashboard recovery\", limit: 10) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Monitoring and Debugging Supabase project troubleshooting unhealthy\", limit: 5) { nodes { title href content } }", + "hasContent": true, + "pages": [] + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"project unhealthy restart pause restore project dashboard inaccessible\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/platform/free-project-pausing", - "title": "Project Pausing" - }, { "url": "https://supabase.com/docs/guides/platform/upgrading", "title": "Upgrading" }, - { - "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", - "title": "Restoring a downloaded backup locally" - }, { "url": "https://supabase.com/docs/guides/platform/delete-project", "title": "Deleting Your Project" }, { - "url": "https://supabase.com/docs/guides/platform/backups", - "title": "Database Backups" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services", - "title": "Project Status reports unhealthy services" - }, - { - "url": "https://supabase.com/docs/guides/database/replication/pipelines-faq", - "title": "Pipelines FAQ" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/restore-project-after-90-days-pause", - "title": "How To Restore a Project Paused for More Than 1 Year" + "url": "https://supabase.com/docs/guides/platform/free-project-pausing", + "title": "Project Pausing" }, { - "url": "https://supabase.com/docs/guides/platform/migrating-within-supabase", - "title": "Migrating within Supabase" + "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", + "title": "Restoring a downloaded backup locally" }, { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" + "url": "https://supabase.com/docs/guides/platform/backups", + "title": "Database Backups" } ], - "resultChars": 70668 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"monitoring debugging project unhealthy database unavailable dashboard troubleshooting\", limit: 8) { nodes { title href content } }", - "hasContent": true, - "pages": [] + "resultChars": 32742 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase changelog breaking change project restart pause restore\", limit: 5) { nodes { title href content } }", + "query": "query { searchDocs(query: \"restart project Supabase dashboard restart unhealthy project\", limit: 5) { nodes { title href content } }", "hasContent": true, "pages": [] }, { "source": "search_docs", - "query": "query { searchDocs(query: \"restart a Supabase project dashboard unhealthy status\", limit: 8) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Monitoring and Debugging\", limit: 3) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services", - "title": "Project Status reports unhealthy services" - }, - { - "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", - "title": "Restoring a downloaded backup locally" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/supabase-dashboard-not-loading-project-not-loading-on-dashboard-LfMq9F", - "title": "Supabase dashboard not loading/ Project not loading on dashboard" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/remove-superuser-access", - "title": "Remove Superuser Access from Studio" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/why-do-i-see-auth--api-requests-in-the-dashboard-my-app-has-no-users-CyadiO", - "title": "Why do I see Auth & API requests in the dashboard? My app has no users" - }, - { - "url": "https://supabase.com/docs/guides/platform/free-project-pausing", - "title": "Project Pausing" + "url": "https://supabase.com/docs/guides/observability/reports", + "title": "Reports" }, { - "url": "https://supabase.com/docs/guides/database/postgres/timeouts", - "title": "Timeouts" + "url": "https://supabase.com/docs/guides/database/postgres/data-deletion", + "title": "Deleting data and dropping objects safely" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/failed-to-run-sql-query-connection-terminated-due-to-connection-timeout", - "title": "Error: Failed to run sql query: Connection terminated due to connection timeout" + "url": "https://supabase.com/docs/guides/database/replication/pipelines-monitoring", + "title": "Monitor pipeline status" } ], - "resultChars": 19697 + "resultChars": 64530 }, { "source": "web_search", - "query": "site:status.supabase.com Supabase status incidents October 2026 dashboard projects", - "pages": [] - }, - { - "source": "web_search", - "query": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services" - } - ] - }, - { - "source": "web_search", - "query": "site:supabase.com/docs/guides/troubleshooting \"Dashboard not loading\" Supabase project not loading", + "query": "site:supabase.com/docs project restart pause restore unhealthy Supabase project", "pages": [] } ] @@ -5499,22 +5498,22 @@ "usage": [ { "model": "gpt-6-luna", - "inputTokens": 89176, - "cacheReadInputTokens": 36235, - "cacheWriteInputTokens": 34715, - "outputTokens": 2191 + "inputTokens": 89426, + "cacheReadInputTokens": 36830, + "cacheWriteInputTokens": 34964, + "outputTokens": 2150 } ], "stepCount": 4, "toolCallCount": 8, - "agentRunDurationMs": 40696, + "agentRunDurationMs": 43337, "sandboxUsage": { - "activeCpuDurationMs": 208252, - "duration": 189359, + "activeCpuDurationMs": 138379, + "duration": 145354, "memory": 8192, "networkTransfer": { - "ingress": 764133561, - "egress": 2803766 + "ingress": 764119563, + "egress": 3675212 } }, "prompt": "My Supabase dashboard says my project is unhealthy, and the dashboard is unusable.\n\nWould restart or pause/restore be better?", @@ -9922,12 +9921,12 @@ ], "suite": "regression", "interface": "mcp", - "passed": true, + "passed": false, "checks": [ { "name": "answered unhealthy project recovery question safely", - "passed": true, - "judgeNotes": "Recommends restarting first, warns against pause/restore for an unhealthy active project, and advises investigating recurring issues and contacting support." + "passed": false, + "judgeNotes": "The answer gives a safe first step and rejects pause/restore, but it describes restore as resuming a paused project rather than distinguishing backup/data restore from a restart." } ], "skills": { @@ -9938,40 +9937,35 @@ "calls": [ { "source": "web_search", - "query": "site:supabase.com/docs project unhealthy restart pause restore Supabase dashboard", + "query": "site:supabase.com/docs restart project pause restore unhealthy project dashboard", "pages": [] }, { "source": "web_search", - "query": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services" - } - ] + "query": "Supabase status dashboard incidents October 6 2026 Supabase status", + "pages": [] } ] }, "usage": [ { "model": "gpt-6-luna", - "inputTokens": 29261, + "inputTokens": 26524, "cacheReadInputTokens": 0, - "cacheWriteInputTokens": 11846, - "outputTokens": 661 + "cacheWriteInputTokens": 11850, + "outputTokens": 794 } ], "stepCount": 1, "toolCallCount": 2, - "agentRunDurationMs": 18209, + "agentRunDurationMs": 20788, "sandboxUsage": { - "activeCpuDurationMs": 160462, - "duration": 132854, + "activeCpuDurationMs": 135693, + "duration": 127165, "memory": 8192, "networkTransfer": { - "ingress": 763059113, - "egress": 2150001 + "ingress": 763231852, + "egress": 2866165 } }, "prompt": "My Supabase dashboard says my project is unhealthy, and the dashboard is unusable.\n\nWould restart or pause/restore be better?", @@ -10003,7 +9997,7 @@ { "name": "answered unhealthy project recovery question safely", "passed": true, - "judgeNotes": "Recommends a restart as a concrete first recovery step, warns against pause/restore for unhealthy services, and advises contacting support if the problem persists. It could more clearly distinguish backup restore from restart." + "judgeNotes": "Recommends restarting rather than pausing or restoring, gives a concrete first step, and directs the user to support if the dashboard remains unusable. It could clarify that restore means backup/data recovery, not restart." } ], "skills": { @@ -10014,12 +10008,12 @@ "calls": [ { "source": "web_search", - "query": "site:supabase.com/docs project unhealthy restart pause restore Supabase project dashboard", + "query": "site:supabase.com/docs project unhealthy restart pause restore dashboard unusable", "pages": [] }, { "source": "web_search", - "query": "Supabase status current incidents dashboard project unhealthy October 6 2026", + "query": "Supabase status current incidents dashboard status page", "pages": [] } ] @@ -10027,22 +10021,22 @@ "usage": [ { "model": "gpt-6-luna", - "inputTokens": 26583, + "inputTokens": 28334, "cacheReadInputTokens": 0, - "cacheWriteInputTokens": 11850, - "outputTokens": 868 + "cacheWriteInputTokens": 11852, + "outputTokens": 951 } ], "stepCount": 1, "toolCallCount": 2, - "agentRunDurationMs": 20266, + "agentRunDurationMs": 20659, "sandboxUsage": { - "activeCpuDurationMs": 218585, - "duration": 174567, + "activeCpuDurationMs": 172888, + "duration": 149310, "memory": 8192, "networkTransfer": { - "ingress": 761492012, - "egress": 1449966 + "ingress": 761747055, + "egress": 1668150 } }, "prompt": "My Supabase dashboard says my project is unhealthy, and the dashboard is unusable.\n\nWould restart or pause/restore be better?", @@ -10074,7 +10068,7 @@ { "name": "answered unhealthy project recovery question safely", "passed": true, - "judgeNotes": "Recommends restarting the database first, clearly distinguishes restart from backup restore, and warns against pausing an unhealthy project. It links Supabase’s unhealthy-services guidance and gives a support escalation path." + "judgeNotes": "Rejects pause/restore as an unhealthy-project fix, recommends a restart as the first recovery step, notes overload may persist, and advises contacting support if the project remains unhealthy." } ], "skills": { @@ -10085,40 +10079,35 @@ "calls": [ { "source": "web_search", - "query": "site:supabase.com/docs project unhealthy restart pause restore Supabase dashboard", + "query": "site:supabase.com/docs project unhealthy restart pause restore dashboard unhealthy", "pages": [] }, { "source": "web_search", - "query": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services" - } - ] + "query": "Supabase status page current incident October 6 2026 dashboard unhealthy", + "pages": [] } ] }, "usage": [ { "model": "gpt-6-luna", - "inputTokens": 29424, + "inputTokens": 24510, "cacheReadInputTokens": 0, "cacheWriteInputTokens": 11846, - "outputTokens": 893 + "outputTokens": 1013 } ], "stepCount": 1, "toolCallCount": 2, - "agentRunDurationMs": 23075, + "agentRunDurationMs": 23500, "sandboxUsage": { - "activeCpuDurationMs": 118932, - "duration": 115334, + "activeCpuDurationMs": 122650, + "duration": 120449, "memory": 8192, "networkTransfer": { - "ingress": 764640332, - "egress": 2934504 + "ingress": 763542280, + "egress": 3064143 } }, "prompt": "My Supabase dashboard says my project is unhealthy, and the dashboard is unusable.\n\nWould restart or pause/restore be better?", From bef65c2943031e0b30267da922055ee22578409a Mon Sep 17 00:00:00 2001 From: Matt Rossman <22670878+mattrossman@users.noreply.github.com> Date: Tue, 6 Oct 2026 17:20:03 -0400 Subject: [PATCH 05/33] Revert "chore: refresh eval results" This reverts commit 03ab19e1bf3dc269f17087d76dd12c8cd5ae630f. --- .../web/src/data/regression-eval-results.json | 375 +++++++++--------- 1 file changed, 193 insertions(+), 182 deletions(-) diff --git a/apps/web/src/data/regression-eval-results.json b/apps/web/src/data/regression-eval-results.json index 2f0fe549..f9d78302 100644 --- a/apps/web/src/data/regression-eval-results.json +++ b/apps/web/src/data/regression-eval-results.json @@ -5132,7 +5132,7 @@ { "name": "answered unhealthy project recovery question safely", "passed": true, - "judgeNotes": "Recommends a database restart as the concrete first step and rejects pause/restore as an unhealthy-project workaround. It could more clearly explain that backup restore is data recovery and advise contacting support if the project remains unhealthy." + "judgeNotes": "Recommends a restart as the first recovery step, distinguishes it from backup restoration, and warns against pause/resume for an unhealthy project. It also points to incident checks and resource-pressure investigation." } ], "skills": { @@ -5149,23 +5149,27 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Monitoring and Debugging Supabase project unhealthy project dashboard\", limit: 5) { nodes { title href content } }", + "query": "query { searchDocs(query: \"Supabase project restart pause restore project dashboard unavailable troubleshooting monitoring debugging\", limit: 5) { nodes { title href content } }", "hasContent": true, "pages": [] }, { "source": "search_docs", - "query": "query { searchDocs(query: \"project unhealthy restart pause restore project dashboard project recovery\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"project unhealthy restart pause restore project recovery dashboard unhealthy\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/platform/free-project-pausing", - "title": "Project Pausing" + "url": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services", + "title": "Project Status reports unhealthy services" }, { "url": "https://supabase.com/docs/guides/platform/delete-project", "title": "Deleting Your Project" }, + { + "url": "https://supabase.com/docs/guides/platform/free-project-pausing", + "title": "Project Pausing" + }, { "url": "https://supabase.com/docs/guides/platform/upgrading", "title": "Upgrading" @@ -5173,79 +5177,46 @@ { "url": "https://supabase.com/docs/guides/platform/backups", "title": "Database Backups" - }, - { - "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", - "title": "Restoring a downloaded backup locally" } ], - "resultChars": 32742 + "resultChars": 32572 }, { - "source": "search_docs", - "query": "query { searchDocs(query: \"Monitoring and Debugging Supabase projects unhealthy status\", limit: 5) { nodes { title href content } }", - "hasContent": true, + "source": "web_search", + "query": "site:supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services restart database unhealthy services", "pages": [] }, { - "source": "search_docs", - "query": "query { searchDocs(query: \"restart project dashboard unhealthy restart project Supabase\", limit: 5) { nodes { title href content } } }", + "source": "web_search", + "query": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", - "title": "Restoring a downloaded backup locally" - }, - { - "url": "https://supabase.com/docs/guides/ai/examples/nextjs-vector-search", - "title": "Vector search with Next.js and OpenAI" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/remove-superuser-access", - "title": "Remove Superuser Access from Studio" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-expo-react-native", - "title": "Build a User Management App with Expo React Native" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-ionic-vue", - "title": "Build a User Management App with Ionic Vue" + "url": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services" } - ], - "resultChars": 89339 - }, - { - "source": "web_search", - "query": "site:supabase.com/docs project unhealthy restart project Supabase restart", - "pages": [] - }, - { - "source": "web_search", - "query": "Supabase status current incidents status.supabase.com October 6 2026", - "pages": [] + ] } ] }, "usage": [ { "model": "gpt-6-luna", - "inputTokens": 87152, - "cacheReadInputTokens": 36795, - "cacheWriteInputTokens": 35584, - "outputTokens": 1566 + "inputTokens": 69751, + "cacheReadInputTokens": 16870, + "cacheWriteInputTokens": 32169, + "outputTokens": 2232 } ], - "stepCount": 4, - "toolCallCount": 9, - "agentRunDurationMs": 29124, + "stepCount": 3, + "toolCallCount": 7, + "agentRunDurationMs": 44205, "sandboxUsage": { - "activeCpuDurationMs": 130367, - "duration": 130067, + "activeCpuDurationMs": 182017, + "duration": 172392, "memory": 8192, "networkTransfer": { - "ingress": 764300671, - "egress": 3850265 + "ingress": 762143255, + "egress": 1795863 } }, "prompt": "My Supabase dashboard says my project is unhealthy, and the dashboard is unusable.\n\nWould restart or pause/restore be better?", @@ -5277,7 +5248,7 @@ { "name": "answered unhealthy project recovery question safely", "passed": true, - "judgeNotes": "Recommends restarting the database first, rejects pause/restore as a troubleshooting workaround, distinguishes it from a routine restart, cites Supabase’s unhealthy-services guidance, and advises contacting support if restarting fails." + "judgeNotes": "Recommends restarting rather than pause/restore, distinguishes the operations, cites Supabase’s unhealthy-services guidance, and directs the user to support if the controls are unavailable. It could more clearly describe restore as data recovery and mention checking logs and resource pressure." } ], "skills": { @@ -5293,73 +5264,47 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"restore paused project Supabase dashboard restart project\", limit: 5) { nodes { title href content } }", - "hasContent": true, - "pages": [] - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Monitoring and Debugging project unhealthy restart project pause restore project\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Supabase project unhealthy restart pause restore project dashboard recovery\", limit: 8) { nodes { title href content } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/platform/upgrading", - "title": "Upgrading" - }, { "url": "https://supabase.com/docs/guides/platform/free-project-pausing", "title": "Project Pausing" }, + { + "url": "https://supabase.com/docs/guides/platform/upgrading", + "title": "Upgrading" + }, { "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", "title": "Restoring a downloaded backup locally" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services", - "title": "Project Status reports unhealthy services" + "url": "https://supabase.com/docs/guides/troubleshooting/restore-project-after-90-days-pause", + "title": "How To Restore a Project Paused for More Than 1 Year" }, - { - "url": "https://supabase.com/docs/guides/platform/delete-project", - "title": "Deleting Your Project" - } - ], - "resultChars": 24720 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"project unhealthy status health check logs restart database Supabase\", limit: 3) { nodes { title href content } }", - "hasContent": true, - "pages": [] - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"pause restore project database backup limitations downtime Supabase\", limit: 3) { nodes { title href content } }", - "hasContent": true, - "pages": [] - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Monitoring and Debugging unhealthy project restart database health checks\", limit: 3) { nodes { title href content } } }", - "hasContent": true, - "pages": [ { "url": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services", "title": "Project Status reports unhealthy services" }, { - "url": "https://supabase.com/docs/guides/observability/detecting", - "title": "Detection checks" + "url": "https://supabase.com/docs/guides/troubleshooting/pausing-pro-projects-vNL-2a", + "title": "Pausing Pro-Projects" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/failed-to-run-sql-query-connection-terminated-due-to-connection-timeout", - "title": "Error: Failed to run sql query: Connection terminated due to connection timeout" + "url": "https://supabase.com/docs/guides/platform/delete-project", + "title": "Deleting Your Project" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres", + "title": "Migrate from Postgres to Supabase" } ], - "resultChars": 14753 + "resultChars": 51101 }, { "source": "web_search", - "query": "site:supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services restart database unhealthy", + "query": "site:supabase.com/docs restart project Supabase dashboard restart project unhealthy", "pages": [] } ] @@ -5367,22 +5312,22 @@ "usage": [ { "model": "gpt-6-luna", - "inputTokens": 77010, - "cacheReadInputTokens": 36918, - "cacheWriteInputTokens": 35659, - "outputTokens": 1703 + "inputTokens": 57617, + "cacheReadInputTokens": 16916, + "cacheWriteInputTokens": 32169, + "outputTokens": 1268 } ], - "stepCount": 4, - "toolCallCount": 7, - "agentRunDurationMs": 30142, + "stepCount": 3, + "toolCallCount": 3, + "agentRunDurationMs": 29120, "sandboxUsage": { - "activeCpuDurationMs": 129484, - "duration": 128214, + "activeCpuDurationMs": 185218, + "duration": 160268, "memory": 8192, "networkTransfer": { - "ingress": 764832046, - "egress": 3719151 + "ingress": 762731208, + "egress": 1934399 } }, "prompt": "My Supabase dashboard says my project is unhealthy, and the dashboard is unusable.\n\nWould restart or pause/restore be better?", @@ -5414,7 +5359,7 @@ { "name": "answered unhealthy project recovery question safely", "passed": true, - "judgeNotes": "Rejects pause/restore as a health reset, distinguishes backup restore from a database restart, recommends a conditional restart for unhealthy services, and advises contacting support if access remains unavailable." + "judgeNotes": "Recommends restart as the first recovery step, warns against pause/restore as a shortcut, and points to workload checks, troubleshooting guidance, and support. It could more clearly explain that backup restore is separate from restart." } ], "skills": { @@ -5423,74 +5368,130 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase" + "supabase", + "supabase-postgres-best-practices" ] }, "docs": { "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Monitoring and Debugging Supabase project troubleshooting unhealthy\", limit: 5) { nodes { title href content } }", - "hasContent": true, - "pages": [] - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"project unhealthy restart pause restore project dashboard inaccessible\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"restart project pause restore unhealthy project dashboard recovery\", limit: 10) { nodes { title href content } } }", "hasContent": true, "pages": [ + { + "url": "https://supabase.com/docs/guides/platform/free-project-pausing", + "title": "Project Pausing" + }, { "url": "https://supabase.com/docs/guides/platform/upgrading", "title": "Upgrading" }, + { + "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", + "title": "Restoring a downloaded backup locally" + }, { "url": "https://supabase.com/docs/guides/platform/delete-project", "title": "Deleting Your Project" }, { - "url": "https://supabase.com/docs/guides/platform/free-project-pausing", - "title": "Project Pausing" + "url": "https://supabase.com/docs/guides/platform/backups", + "title": "Database Backups" }, { - "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", - "title": "Restoring a downloaded backup locally" + "url": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services", + "title": "Project Status reports unhealthy services" }, { - "url": "https://supabase.com/docs/guides/platform/backups", - "title": "Database Backups" + "url": "https://supabase.com/docs/guides/database/replication/pipelines-faq", + "title": "Pipelines FAQ" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/restore-project-after-90-days-pause", + "title": "How To Restore a Project Paused for More Than 1 Year" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-within-supabase", + "title": "Migrating within Supabase" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" } ], - "resultChars": 32742 + "resultChars": 70668 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"monitoring debugging project unhealthy database unavailable dashboard troubleshooting\", limit: 8) { nodes { title href content } }", + "hasContent": true, + "pages": [] }, { "source": "search_docs", - "query": "query { searchDocs(query: \"restart project Supabase dashboard restart unhealthy project\", limit: 5) { nodes { title href content } }", + "query": "query { searchDocs(query: \"Supabase changelog breaking change project restart pause restore\", limit: 5) { nodes { title href content } }", "hasContent": true, "pages": [] }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Monitoring and Debugging\", limit: 3) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"restart a Supabase project dashboard unhealthy status\", limit: 8) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/observability/reports", - "title": "Reports" + "url": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services", + "title": "Project Status reports unhealthy services" + }, + { + "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", + "title": "Restoring a downloaded backup locally" }, { - "url": "https://supabase.com/docs/guides/database/postgres/data-deletion", - "title": "Deleting data and dropping objects safely" + "url": "https://supabase.com/docs/guides/troubleshooting/supabase-dashboard-not-loading-project-not-loading-on-dashboard-LfMq9F", + "title": "Supabase dashboard not loading/ Project not loading on dashboard" }, { - "url": "https://supabase.com/docs/guides/database/replication/pipelines-monitoring", - "title": "Monitor pipeline status" + "url": "https://supabase.com/docs/guides/self-hosting/remove-superuser-access", + "title": "Remove Superuser Access from Studio" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/why-do-i-see-auth--api-requests-in-the-dashboard-my-app-has-no-users-CyadiO", + "title": "Why do I see Auth & API requests in the dashboard? My app has no users" + }, + { + "url": "https://supabase.com/docs/guides/platform/free-project-pausing", + "title": "Project Pausing" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/timeouts", + "title": "Timeouts" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/failed-to-run-sql-query-connection-terminated-due-to-connection-timeout", + "title": "Error: Failed to run sql query: Connection terminated due to connection timeout" } ], - "resultChars": 64530 + "resultChars": 19697 }, { "source": "web_search", - "query": "site:supabase.com/docs project restart pause restore unhealthy Supabase project", + "query": "site:status.supabase.com Supabase status incidents October 2026 dashboard projects", + "pages": [] + }, + { + "source": "web_search", + "query": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services" + } + ] + }, + { + "source": "web_search", + "query": "site:supabase.com/docs/guides/troubleshooting \"Dashboard not loading\" Supabase project not loading", "pages": [] } ] @@ -5498,22 +5499,22 @@ "usage": [ { "model": "gpt-6-luna", - "inputTokens": 89426, - "cacheReadInputTokens": 36830, - "cacheWriteInputTokens": 34964, - "outputTokens": 2150 + "inputTokens": 89176, + "cacheReadInputTokens": 36235, + "cacheWriteInputTokens": 34715, + "outputTokens": 2191 } ], "stepCount": 4, "toolCallCount": 8, - "agentRunDurationMs": 43337, + "agentRunDurationMs": 40696, "sandboxUsage": { - "activeCpuDurationMs": 138379, - "duration": 145354, + "activeCpuDurationMs": 208252, + "duration": 189359, "memory": 8192, "networkTransfer": { - "ingress": 764119563, - "egress": 3675212 + "ingress": 764133561, + "egress": 2803766 } }, "prompt": "My Supabase dashboard says my project is unhealthy, and the dashboard is unusable.\n\nWould restart or pause/restore be better?", @@ -9921,12 +9922,12 @@ ], "suite": "regression", "interface": "mcp", - "passed": false, + "passed": true, "checks": [ { "name": "answered unhealthy project recovery question safely", - "passed": false, - "judgeNotes": "The answer gives a safe first step and rejects pause/restore, but it describes restore as resuming a paused project rather than distinguishing backup/data restore from a restart." + "passed": true, + "judgeNotes": "Recommends restarting first, warns against pause/restore for an unhealthy active project, and advises investigating recurring issues and contacting support." } ], "skills": { @@ -9937,35 +9938,40 @@ "calls": [ { "source": "web_search", - "query": "site:supabase.com/docs restart project pause restore unhealthy project dashboard", + "query": "site:supabase.com/docs project unhealthy restart pause restore Supabase dashboard", "pages": [] }, { "source": "web_search", - "query": "Supabase status dashboard incidents October 6 2026 Supabase status", - "pages": [] + "query": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services" + } + ] } ] }, "usage": [ { "model": "gpt-6-luna", - "inputTokens": 26524, + "inputTokens": 29261, "cacheReadInputTokens": 0, - "cacheWriteInputTokens": 11850, - "outputTokens": 794 + "cacheWriteInputTokens": 11846, + "outputTokens": 661 } ], "stepCount": 1, "toolCallCount": 2, - "agentRunDurationMs": 20788, + "agentRunDurationMs": 18209, "sandboxUsage": { - "activeCpuDurationMs": 135693, - "duration": 127165, + "activeCpuDurationMs": 160462, + "duration": 132854, "memory": 8192, "networkTransfer": { - "ingress": 763231852, - "egress": 2866165 + "ingress": 763059113, + "egress": 2150001 } }, "prompt": "My Supabase dashboard says my project is unhealthy, and the dashboard is unusable.\n\nWould restart or pause/restore be better?", @@ -9997,7 +10003,7 @@ { "name": "answered unhealthy project recovery question safely", "passed": true, - "judgeNotes": "Recommends restarting rather than pausing or restoring, gives a concrete first step, and directs the user to support if the dashboard remains unusable. It could clarify that restore means backup/data recovery, not restart." + "judgeNotes": "Recommends a restart as a concrete first recovery step, warns against pause/restore for unhealthy services, and advises contacting support if the problem persists. It could more clearly distinguish backup restore from restart." } ], "skills": { @@ -10008,12 +10014,12 @@ "calls": [ { "source": "web_search", - "query": "site:supabase.com/docs project unhealthy restart pause restore dashboard unusable", + "query": "site:supabase.com/docs project unhealthy restart pause restore Supabase project dashboard", "pages": [] }, { "source": "web_search", - "query": "Supabase status current incidents dashboard status page", + "query": "Supabase status current incidents dashboard project unhealthy October 6 2026", "pages": [] } ] @@ -10021,22 +10027,22 @@ "usage": [ { "model": "gpt-6-luna", - "inputTokens": 28334, + "inputTokens": 26583, "cacheReadInputTokens": 0, - "cacheWriteInputTokens": 11852, - "outputTokens": 951 + "cacheWriteInputTokens": 11850, + "outputTokens": 868 } ], "stepCount": 1, "toolCallCount": 2, - "agentRunDurationMs": 20659, + "agentRunDurationMs": 20266, "sandboxUsage": { - "activeCpuDurationMs": 172888, - "duration": 149310, + "activeCpuDurationMs": 218585, + "duration": 174567, "memory": 8192, "networkTransfer": { - "ingress": 761747055, - "egress": 1668150 + "ingress": 761492012, + "egress": 1449966 } }, "prompt": "My Supabase dashboard says my project is unhealthy, and the dashboard is unusable.\n\nWould restart or pause/restore be better?", @@ -10068,7 +10074,7 @@ { "name": "answered unhealthy project recovery question safely", "passed": true, - "judgeNotes": "Rejects pause/restore as an unhealthy-project fix, recommends a restart as the first recovery step, notes overload may persist, and advises contacting support if the project remains unhealthy." + "judgeNotes": "Recommends restarting the database first, clearly distinguishes restart from backup restore, and warns against pausing an unhealthy project. It links Supabase’s unhealthy-services guidance and gives a support escalation path." } ], "skills": { @@ -10079,35 +10085,40 @@ "calls": [ { "source": "web_search", - "query": "site:supabase.com/docs project unhealthy restart pause restore dashboard unhealthy", + "query": "site:supabase.com/docs project unhealthy restart pause restore Supabase dashboard", "pages": [] }, { "source": "web_search", - "query": "Supabase status page current incident October 6 2026 dashboard unhealthy", - "pages": [] + "query": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services" + } + ] } ] }, "usage": [ { "model": "gpt-6-luna", - "inputTokens": 24510, + "inputTokens": 29424, "cacheReadInputTokens": 0, "cacheWriteInputTokens": 11846, - "outputTokens": 1013 + "outputTokens": 893 } ], "stepCount": 1, "toolCallCount": 2, - "agentRunDurationMs": 23500, + "agentRunDurationMs": 23075, "sandboxUsage": { - "activeCpuDurationMs": 122650, - "duration": 120449, + "activeCpuDurationMs": 118932, + "duration": 115334, "memory": 8192, "networkTransfer": { - "ingress": 763542280, - "egress": 3064143 + "ingress": 764640332, + "egress": 2934504 } }, "prompt": "My Supabase dashboard says my project is unhealthy, and the dashboard is unusable.\n\nWould restart or pause/restore be better?", From c7c9e5cd071c7e09ac5ad84ab40eb68073318003 Mon Sep 17 00:00:00 2001 From: Matt Rossman <22670878+mattrossman@users.noreply.github.com> Date: Tue, 6 Oct 2026 17:20:03 -0400 Subject: [PATCH 06/33] Revert "chore: temporary eval change for refresh pilot" This reverts commit 9c94bd1fe4e4d9ecef0fc3169297f40947cf0e80. --- .../resolve-reliability-001-unhealthy-project-recovery/EVAL.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/evals/regression/resolve-reliability-001-unhealthy-project-recovery/EVAL.ts b/evals/regression/resolve-reliability-001-unhealthy-project-recovery/EVAL.ts index d9c492ac..d1382da7 100644 --- a/evals/regression/resolve-reliability-001-unhealthy-project-recovery/EVAL.ts +++ b/evals/regression/resolve-reliability-001-unhealthy-project-recovery/EVAL.ts @@ -35,5 +35,3 @@ const scorer: ToolScorer = async (ctx) => { }; export default scorer; - -// Temporary change to exercise the refresh results commit. From 7cfedb1821296acd56c30677f93126ef59e407a3 Mon Sep 17 00:00:00 2001 From: Matt Rossman <22670878+mattrossman@users.noreply.github.com> Date: Tue, 6 Oct 2026 17:20:03 -0400 Subject: [PATCH 07/33] fix(ci): give the test job an OpenAI key for the smoke test judge --- .github/workflows/test.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index de7de978..b25b6806 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -28,6 +28,9 @@ jobs: run: pnpm install --frozen-lockfile - name: Run checks + # The framework smoke test calls the default LLM judge. Fork PRs get no secrets, so it fails there. + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} run: pnpm check # Own job since it's the slowest suite. Every test boots fresh PGlite databases. From 4f331d9c16cddb5a6a0467e4ed5d44e2a228207c Mon Sep 17 00:00:00 2001 From: Matt Rossman <22670878+mattrossman@users.noreply.github.com> Date: Wed, 7 Oct 2026 09:43:20 -0400 Subject: [PATCH 08/33] test(core): drop the duplicate Codex tool call id after merging main --- packages/core/src/agents/codex/parser.test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/core/src/agents/codex/parser.test.ts b/packages/core/src/agents/codex/parser.test.ts index af6fa10f..f126c8a8 100644 --- a/packages/core/src/agents/codex/parser.test.ts +++ b/packages/core/src/agents/codex/parser.test.ts @@ -232,7 +232,6 @@ describe('codexParser', () => { const adapted = adaptTranscript(codexParser.parseTranscript(stream).events); expect(adapted.transcript[0]).toEqual({ type: 'tool_call', - id: 'item_7', name: 'execute_sql', id: 'item_7', input: args, From f29f99d4afe07a1818590f616cdbdb6e3c5b27b4 Mon Sep 17 00:00:00 2001 From: Matt Rossman <22670878+mattrossman@users.noreply.github.com> Date: Wed, 7 Oct 2026 09:43:20 -0400 Subject: [PATCH 09/33] chore: upgrade vitest to 5 and name each test run --- apps/framework/package.json | 2 +- apps/framework/vitest.cli-lib.config.ts | 3 + apps/framework/vitest.config.ts | 3 + apps/web/vite.config.ts | 4 + packages/core/vitest.config.ts | 3 + packages/platform-lite/vitest.config.ts | 1 + packages/sandbox/vitest.config.ts | 3 + pnpm-lock.yaml | 261 +++++++++--------------- pnpm-workspace.yaml | 2 +- 9 files changed, 119 insertions(+), 163 deletions(-) create mode 100644 apps/framework/vitest.cli-lib.config.ts create mode 100644 apps/framework/vitest.config.ts create mode 100644 packages/core/vitest.config.ts create mode 100644 packages/sandbox/vitest.config.ts diff --git a/apps/framework/package.json b/apps/framework/package.json index e2922fc3..aefb7e20 100644 --- a/apps/framework/package.json +++ b/apps/framework/package.json @@ -14,7 +14,7 @@ "test": "vitest run", "test:framework": "node --env-file-if-exists=../../.env --import tsx/esm scripts/smoke-framework.ts", "test:vercel-runner": "vitest run scripts/run-vercel-evals.test.ts scripts/export-results.test.ts lib/cli-args.test.ts lib/experiment-files.test.ts lib/sample-sets.test.ts", - "test:cli-lib": "vitest run --root ../.. --passWithNoTests experiments/cli evals/cli", + "test:cli-lib": "vitest run --config vitest.cli-lib.config.ts --passWithNoTests experiments/cli evals/cli", "export-results": "node --import tsx/esm scripts/export-results.ts", "demo:mcp": "node --env-file=../../.env --import tsx/esm scripts/mcp-demo.ts", "demo:executor": "node --env-file=../../.env --import tsx/esm scripts/executor-demo.ts", diff --git a/apps/framework/vitest.cli-lib.config.ts b/apps/framework/vitest.cli-lib.config.ts new file mode 100644 index 00000000..571d0354 --- /dev/null +++ b/apps/framework/vitest.cli-lib.config.ts @@ -0,0 +1,3 @@ +import { defineConfig } from 'vitest/config'; + +export default defineConfig({ test: { name: 'cli evals', root: '../..' } }); diff --git a/apps/framework/vitest.config.ts b/apps/framework/vitest.config.ts new file mode 100644 index 00000000..a0bac02f --- /dev/null +++ b/apps/framework/vitest.config.ts @@ -0,0 +1,3 @@ +import { defineConfig } from 'vitest/config'; + +export default defineConfig({ test: { name: 'framework' } }); diff --git a/apps/web/vite.config.ts b/apps/web/vite.config.ts index 9d93680d..1312e189 100644 --- a/apps/web/vite.config.ts +++ b/apps/web/vite.config.ts @@ -1,3 +1,4 @@ +/// import path from "path" import tailwindcss from "@tailwindcss/vite" import react from "@vitejs/plugin-react" @@ -21,4 +22,7 @@ export default defineConfig({ "@": path.resolve(__dirname, "./src"), }, }, + test: { + name: "web", + }, }) diff --git a/packages/core/vitest.config.ts b/packages/core/vitest.config.ts new file mode 100644 index 00000000..3ce68b17 --- /dev/null +++ b/packages/core/vitest.config.ts @@ -0,0 +1,3 @@ +import { defineConfig } from 'vitest/config'; + +export default defineConfig({ test: { name: 'core' } }); diff --git a/packages/platform-lite/vitest.config.ts b/packages/platform-lite/vitest.config.ts index e2a9474f..ada5da89 100644 --- a/packages/platform-lite/vitest.config.ts +++ b/packages/platform-lite/vitest.config.ts @@ -2,6 +2,7 @@ import { defineConfig } from 'vitest/config'; export default defineConfig({ test: { + name: 'platform-lite', environment: 'node', testTimeout: 30000, }, diff --git a/packages/sandbox/vitest.config.ts b/packages/sandbox/vitest.config.ts new file mode 100644 index 00000000..0e853ec5 --- /dev/null +++ b/packages/sandbox/vitest.config.ts @@ -0,0 +1,3 @@ +import { defineConfig } from 'vitest/config'; + +export default defineConfig({ test: { name: 'sandbox' } }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9c5a0fb3..4fac864d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -52,8 +52,8 @@ catalogs: specifier: ^7.3.1 version: 7.3.5 vitest: - specifier: ^4.1.5 - version: 4.1.8 + specifier: ^5.0.3 + version: 5.0.3 zod: specifier: ^4.4.3 version: 4.4.3 @@ -164,7 +164,7 @@ importers: version: 7.3.5(@types/node@22.19.20)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4) vitest: specifier: 'catalog:' - version: 4.1.8(@opentelemetry/api@1.9.1)(@types/node@22.19.20)(happy-dom@20.10.2)(vite@7.3.5(@types/node@22.19.20)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4)) + version: 5.0.3(@opentelemetry/api@1.9.1)(@types/node@22.19.20)(happy-dom@20.10.2)(vite@7.3.5(@types/node@22.19.20)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4)) zod: specifier: 'catalog:' version: 4.4.3 @@ -280,7 +280,7 @@ importers: version: 7.3.5(@types/node@22.19.20)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4) vitest: specifier: 'catalog:' - version: 4.1.8(@opentelemetry/api@1.9.1)(@types/node@22.19.20)(happy-dom@20.10.2)(vite@7.3.5(@types/node@22.19.20)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4)) + version: 5.0.3(@opentelemetry/api@1.9.1)(@types/node@22.19.20)(happy-dom@20.10.2)(vite@7.3.5(@types/node@22.19.20)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4)) packages/core: dependencies: @@ -323,7 +323,7 @@ importers: version: 1.18.31 vitest: specifier: 'catalog:' - version: 4.1.8(@opentelemetry/api@1.9.1)(@types/node@25.9.3)(happy-dom@20.10.2)(vite@7.3.5(@types/node@25.9.3)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4)) + version: 5.0.3(@opentelemetry/api@1.9.1)(@types/node@25.9.3)(happy-dom@20.10.2)(vite@7.3.5(@types/node@25.9.3)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4)) packages/platform-lite: dependencies: @@ -369,7 +369,7 @@ importers: version: 5.9.3 vitest: specifier: 'catalog:' - version: 4.1.8(@opentelemetry/api@1.9.1)(@types/node@22.19.20)(happy-dom@20.10.2)(vite@7.3.5(@types/node@22.19.20)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4)) + version: 5.0.3(@opentelemetry/api@1.9.1)(@types/node@22.19.20)(happy-dom@20.10.2)(vite@7.3.5(@types/node@22.19.20)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4)) packages/sandbox: dependencies: @@ -391,7 +391,7 @@ importers: version: 5.9.3 vitest: specifier: 'catalog:' - version: 4.1.8(@opentelemetry/api@1.9.1)(@types/node@22.19.20)(happy-dom@20.10.2)(vite@7.3.5(@types/node@22.19.20)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4)) + version: 5.0.3(@opentelemetry/api@1.9.1)(@types/node@22.19.20)(happy-dom@20.10.2)(vite@7.3.5(@types/node@22.19.20)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4)) packages: @@ -1271,6 +1271,9 @@ packages: '@jridgewell/sourcemap-codec@1.5.5': resolution: {integrity: sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==} + '@jridgewell/sourcemap-codec@1.6.0': + resolution: {integrity: sha512-T7jf+5zgsZHwNJ4lvQ7/aezbyk0nNX+zJVWpmHA7VYsEx7a7qr5Rg5IbtJFqkgze5Y2sruq1RUY8Q837Od7iFw==} + '@jridgewell/trace-mapping@0.3.31': resolution: {integrity: sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==} @@ -2617,11 +2620,8 @@ packages: peerDependencies: vite: ^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0 - '@vitest/expect@4.1.8': - resolution: {integrity: sha512-h3nDO677RDLEGlBxyQ5CW8RlMThSKSRLUePLOx09gNIWRL40edgA1GCZSZgf1W55MFAG6/Sw14KeaAnqv0NKdQ==} - - '@vitest/mocker@4.1.8': - resolution: {integrity: sha512-LEiN/xe4OSIbKe9HQIp5OC24agGD9J5CnmMgsLohVVoOPWL9a2sBoR6VBx43jQZb7Kr1l4RCuyCJzcAa0+dojw==} + '@vitest/mocker@5.0.3': + resolution: {integrity: sha512-T8sWAIbkSyAjkwTcaEc3Iu0o9A27X1/kdXrizhZkGuSKScRQtRzclfAMpOTcGdXCsqxeWlpGy3XjqaW8CpLORg==} peerDependencies: msw: ^2.4.9 vite: ^6.0.0 || ^7.0.0 || ^8.0.0 @@ -2631,20 +2631,8 @@ packages: vite: optional: true - '@vitest/pretty-format@4.1.8': - resolution: {integrity: sha512-9GasEBxpZ1VYIpqHf/0+YGg121uSNwCKOJqIrTwWP/TB7DmFCiaBpNl3aPZzoLWfWkuqhbH8vJIVobZkvdo2cA==} - - '@vitest/runner@4.1.8': - resolution: {integrity: sha512-EmVxeBAfMJvycdjd6Hm+RbFBbA9fKvo0Kx37hNpBYoYeavH3RNsBXWDooR1mgD52dCrxIIuP7UotpfiwOikvcg==} - - '@vitest/snapshot@4.1.8': - resolution: {integrity: sha512-acfZboRmAIf05DEKcBQy33VXojFJjtUdLyo7oOmV9kebb2xdU01UknNiPuPZoJZQyO7DF0gZdTGTpeAzET9QPQ==} - - '@vitest/spy@4.1.8': - resolution: {integrity: sha512-6EevtBp6OZOPF7bmz36HrGMeP3txgVSrgebWxHOafDXGkhIzfXK14f8KF6MuFfgXXUeHxmpD3BQxkV00/3s5mA==} - - '@vitest/utils@4.1.8': - resolution: {integrity: sha512-uOJamYALNhfJ6iolExyQM40yIQwDqYnkKtQ5VCiSe17E33H0aQ/u+1GlRuz4LZBk6Mm3sg90G9hEbmEt37C1Zg==} + '@vitest/spy@5.0.3': + resolution: {integrity: sha512-XhFysQTB8AZ+P4gMi+Lpo99vg2AZi0qKpaB9yXQl37+CaMEAPO3iH/wGVnSyL5MPERiLezpqTVtrR6UZH5GCXg==} '@workflow/serde@4.1.0-beta.2': resolution: {integrity: sha512-8kkeoQKLDaKXefjV5dbhBj2aErfKp1Mc4pb6tj8144cF+Em5SPbyMbyLCHp+BVrFfFVCBluCtMx+jjvaFVZGww==} @@ -3080,8 +3068,8 @@ packages: resolution: {integrity: sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==} engines: {node: '>= 0.4'} - es-module-lexer@2.1.0: - resolution: {integrity: sha512-n27zTYMjYu1aj4MjCWzSP7G9r75utsaoc8m61weK+W8JMBGGQybd43GstCXZ3WNmSFtGT9wi59qQTW6mhTR5LQ==} + es-module-lexer@2.3.2: + resolution: {integrity: sha512-poHGpORABojJJucnV9KbOavETW8lBVnphkW77ER5/BQ5Fz7oXSoCNek7IH3vR5nRjdsEz926ibFYX8KtLQmdyw==} es-object-atoms@1.1.2: resolution: {integrity: sha512-HWcBoN6NileqtSydK2FqHbS/LoDd2pqrnQHLyJzBj4kOp/ky2MWMN694xOfkK8/SnUsW2DH7EfyVlydKCsm1Zw==} @@ -3251,8 +3239,8 @@ packages: cpu: [x64] os: [win32] - expect-type@1.3.0: - resolution: {integrity: sha512-knvyeauYhqjOYvQ66MznSMs83wmHrCycNEN6Ao+2AeYEfxUIkuiVxdEa1qlGEPK+We3n0THiDciYSsCcgW/DoA==} + expect-type@1.4.0: + resolution: {integrity: sha512-KfYbmpRm0VbLjEvVa9yGwCi9GI34xvi7A/HXYWQO65CSD2u3MczUJSuwXKFIxlGsgBQizV9q5J9NHj4VG0n+pA==} engines: {node: '>=12.0.0'} express-rate-limit@8.5.2: @@ -3816,6 +3804,9 @@ packages: magic-string@0.30.21: resolution: {integrity: sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==} + magic-string@1.4.3: + resolution: {integrity: sha512-z12OxmaPGE0F4xlpGdjuAUckipLpQlTAuAmyGhNoD7ODpYUzvDfvtxUbjM/jwznyP178oju/KDdyi220wNJ0RQ==} + math-intrinsics@1.1.0: resolution: {integrity: sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==} engines: {node: '>= 0.4'} @@ -3960,8 +3951,8 @@ packages: resolution: {integrity: sha512-EFVjAYfzWqWsBMRHPMAXLCDIJnpMhdWAqR7xG6M6a2cs6PMFpl/+Z20w9zDW4vkxOFfddegBKq9Rehd0bxWE7A==} engines: {node: '>= 10'} - obug@2.1.2: - resolution: {integrity: sha512-AWGB9WFcRXOQs48Z/udjI5ZcZMHXwX8XPByNpOydgcGsDLIzjGizhoMWJyKAWze7AVW/2W1i+/gPX4YtKe5cyg==} + obug@2.2.1: + resolution: {integrity: sha512-XrsrhT5sybtKI6wakr2SPOlGZWWYbUXZ7a0jT8/QOeAPau+1X/bSegNe5YR75oJmEZQbKningirmGOEJCIk61Q==} engines: {node: '>=12.20.0'} on-finished@2.4.1: @@ -4072,9 +4063,6 @@ packages: path-to-regexp@8.4.2: resolution: {integrity: sha512-qRcuIdP69NPm4qbACK+aDogI5CBDMi1jKe0ry5rSQJz8JVLsC7jV8XpiJjGRLLol3N+R5ihGYcrPLTno6pAdBA==} - pathe@2.0.3: - resolution: {integrity: sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==} - pg-cloudflare@1.4.0: resolution: {integrity: sha512-Vo7z/6rrQYxpNRylp4Tlob2elzbh+N/MOQbxFVWCxS7oEx6jF53GTJFxK2WWpKuBRkmiin4Mt+xofFDjx09R0A==} @@ -4126,6 +4114,10 @@ packages: resolution: {integrity: sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==} engines: {node: '>=12'} + picomatch@4.0.7: + resolution: {integrity: sha512-qcJu88Q2IWqJsDD529JKMdwGm/dvInW4HvQnRwiH9JtihJvzGOscDtHE3x1pBKeUOTysQ8kVmLnJ2kJu7yhcGA==} + engines: {node: '>=12'} + pkce-challenge@5.0.1: resolution: {integrity: sha512-wQ0b/W4Fr01qtpHlqSqspcj3EhBvimsdh0KlHhH8HRZnMsEa0ea2fTULOXOS9ccQr3om+GcGRk4e+isrZWV8qQ==} engines: {node: '>=16.20.0'} @@ -4437,9 +4429,6 @@ packages: resolution: {integrity: sha512-6x6dK6zJdpTzF4sQeNYxwtvBzf6Eg4GtlesS94HOvTudUeyK2WXAaIfmDgsyslYrRBeFIlsi54AYsFGUuhmvrQ==} engines: {node: '>= 0.4'} - siginfo@2.0.0: - resolution: {integrity: sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==} - signal-exit@3.0.7: resolution: {integrity: sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==} @@ -4469,9 +4458,6 @@ packages: sprintf-js@1.0.3: resolution: {integrity: sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==} - stackback@0.0.2: - resolution: {integrity: sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==} - standardwebhooks@1.0.0: resolution: {integrity: sha512-BbHGOQK9olHPMvQNHWul6MYlrRTAOKn03rOe4A8O3CLWhNf4YHBqq2HJKKC+sfqpxiBY52pNeesD6jIiLDz8jg==} @@ -4479,8 +4465,8 @@ packages: resolution: {integrity: sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==} engines: {node: '>= 0.8'} - std-env@4.1.0: - resolution: {integrity: sha512-Rq7ybcX2RuC55r9oaPVEW7/xu3tj8u4GeBYHBWCychFtzMIr86A7e3PPEBPT37sHStKX3+TiX/Fr/ACmJLVlLQ==} + std-env@4.3.0: + resolution: {integrity: sha512-OtU/EgQ1kIm5KwqQpBC6ZEMXrZRui11w8zgfTWp8cdO9B8OaPsbA8bTHO2P+HNo1VlUTGMVBwPhydu6poeXiag==} stdin-discarder@0.2.2: resolution: {integrity: sha512-UhDfHmA92YAlNnCfhmq0VeNL5bDbiZGg7sZ2IvPsXubGkiNa9EC+tUTsjBRsYUAz87btI6/1wf4XoVvQ3uRnmQ==} @@ -4564,21 +4550,18 @@ packages: tiny-invariant@1.3.3: resolution: {integrity: sha512-+FbBPE1o9QAYvviau/qC5SE3caw21q3xkvWKBtja5vgqOWIHHJ3ioaq1VPfn/Szqctz2bU/oYeKd9/z5BL+PVg==} - tinybench@2.9.0: - resolution: {integrity: sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==} + tinybench@6.2.1: + resolution: {integrity: sha512-IAbQaPJIIdhVxi0mqy9lez1wEwje0UQf9F1vwtbelEG5Nbgn0nPOxCvU+mNeNS9gyJTAhrHp74CbUaSynQpZcQ==} + engines: {node: '>=20.0.0'} - tinyexec@1.2.4: - resolution: {integrity: sha512-SHf/r48b7vOrjve9PxJo3MN5v5yuyjHvdUcrQffT3WXMUfnGmHDVbC4k3sHJaJTgZCwpUplIaAo5ANtMyp3YHg==} + tinyexec@1.3.1: + resolution: {integrity: sha512-GCvB3aoys96IuDFBMcTB46JOR6mdMtAToqwiW8JlWhsoh1mhHi/xn9ss/Dg7N555GiJyEt2qzoG/NHCwM6h1EA==} engines: {node: '>=18'} tinyglobby@0.2.17: resolution: {integrity: sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==} engines: {node: '>=12.0.0'} - tinyrainbow@3.1.0: - resolution: {integrity: sha512-Bf+ILmBgretUrdJxzXM0SgXLZ3XfiaUuOj/IKQHuTXip+05Xn+uyEYdVg0kYDipTBcLrCVyUzAPz7QmArb0mmw==} - engines: {node: '>=14.0.0'} - to-regex-range@5.0.1: resolution: {integrity: sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==} engines: {node: '>=8.0'} @@ -4755,23 +4738,23 @@ packages: yaml: optional: true - vitest@4.1.8: - resolution: {integrity: sha512-flY6ScbCIt9HThs+C5HS7jvGOB560DJtk/Z15IQROTA6zEy49Nh8T/dofWTQL+n3vswqn87sbJNiuqw1SDp5Ig==} - engines: {node: ^20.0.0 || ^22.0.0 || >=24.0.0} + vitest@5.0.3: + resolution: {integrity: sha512-xMw97S3rjdtj5dkVat7jCsqWBpvchs3RlpQctUqwJD0KkERk40vz2fJ77lDwW/Vzh/pk18eItYAzkodhSes3jQ==} + engines: {node: ^22.12.0 || ^24.0.0 || >=26.0.0} hasBin: true peerDependencies: '@edge-runtime/vm': '*' '@opentelemetry/api': ^1.9.0 - '@types/node': ^20.0.0 || ^22.0.0 || >=24.0.0 - '@vitest/browser-playwright': 4.1.8 - '@vitest/browser-preview': 4.1.8 - '@vitest/browser-webdriverio': 4.1.8 - '@vitest/coverage-istanbul': 4.1.8 - '@vitest/coverage-v8': 4.1.8 - '@vitest/ui': 4.1.8 + '@types/node': ^22.0.0 || >=24.0.0 + '@vitest/browser-playwright': 5.0.3 + '@vitest/browser-preview': 5.0.3 + '@vitest/browser-webdriverio': ^5.0.0-beta.5 || >=5.0.0 + '@vitest/coverage-istanbul': 5.0.3 + '@vitest/coverage-v8': 5.0.3 + '@vitest/ui': 5.0.3 happy-dom: '*' jsdom: '*' - vite: ^6.0.0 || ^7.0.0 || ^8.0.0 + vite: ^6.4.0 || ^7.0.0 || ^8.0.0 peerDependenciesMeta: '@edge-runtime/vm': optional: true @@ -4817,9 +4800,9 @@ packages: engines: {node: ^16.13.0 || >=18.0.0} hasBin: true - why-is-node-running@2.3.0: - resolution: {integrity: sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w==} - engines: {node: '>=8'} + why-is-node-running@3.2.1: + resolution: {integrity: sha512-Tb2FUhB4vUsGQlfSquQLYkApkuPAFQXGFzxWKHHumVz2dK+X1RUm/HnID4+TfIGYJ1kTcwOaCk/buYCEJr6YjQ==} + engines: {node: '>=20.11'} hasBin: true word-wrap@1.2.5: @@ -5570,6 +5553,8 @@ snapshots: '@jridgewell/sourcemap-codec@1.5.5': {} + '@jridgewell/sourcemap-codec@1.6.0': {} + '@jridgewell/trace-mapping@0.3.31': dependencies: '@jridgewell/resolve-uri': 3.1.2 @@ -6967,54 +6952,25 @@ snapshots: transitivePeerDependencies: - supports-color - '@vitest/expect@4.1.8': - dependencies: - '@standard-schema/spec': 1.1.0 - '@types/chai': 5.2.3 - '@vitest/spy': 4.1.8 - '@vitest/utils': 4.1.8 - chai: 6.2.2 - tinyrainbow: 3.1.0 - - '@vitest/mocker@4.1.8(vite@7.3.5(@types/node@22.19.20)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4))': + '@vitest/mocker@5.0.3(vite@7.3.5(@types/node@22.19.20)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4))': dependencies: - '@vitest/spy': 4.1.8 + '@jridgewell/trace-mapping': 0.3.31 + '@vitest/spy': 5.0.3 estree-walker: 3.0.3 - magic-string: 0.30.21 + magic-string: 1.4.3 optionalDependencies: vite: 7.3.5(@types/node@22.19.20)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4) - '@vitest/mocker@4.1.8(vite@7.3.5(@types/node@25.9.3)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4))': + '@vitest/mocker@5.0.3(vite@7.3.5(@types/node@25.9.3)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4))': dependencies: - '@vitest/spy': 4.1.8 + '@jridgewell/trace-mapping': 0.3.31 + '@vitest/spy': 5.0.3 estree-walker: 3.0.3 - magic-string: 0.30.21 + magic-string: 1.4.3 optionalDependencies: vite: 7.3.5(@types/node@25.9.3)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4) - '@vitest/pretty-format@4.1.8': - dependencies: - tinyrainbow: 3.1.0 - - '@vitest/runner@4.1.8': - dependencies: - '@vitest/utils': 4.1.8 - pathe: 2.0.3 - - '@vitest/snapshot@4.1.8': - dependencies: - '@vitest/pretty-format': 4.1.8 - '@vitest/utils': 4.1.8 - magic-string: 0.30.21 - pathe: 2.0.3 - - '@vitest/spy@4.1.8': {} - - '@vitest/utils@4.1.8': - dependencies: - '@vitest/pretty-format': 4.1.8 - convert-source-map: 2.0.0 - tinyrainbow: 3.1.0 + '@vitest/spy@5.0.3': {} '@workflow/serde@4.1.0-beta.2': {} @@ -7401,7 +7357,7 @@ snapshots: es-errors@1.3.0: {} - es-module-lexer@2.1.0: {} + es-module-lexer@2.3.2: {} es-object-atoms@1.1.2: dependencies: @@ -7669,7 +7625,7 @@ snapshots: executor@1.4.29-windows-x64: optional: true - expect-type@1.3.0: {} + expect-type@1.4.0: {} express-rate-limit@8.5.2(express@5.2.1): dependencies: @@ -8160,6 +8116,10 @@ snapshots: dependencies: '@jridgewell/sourcemap-codec': 1.5.5 + magic-string@1.4.3: + dependencies: + '@jridgewell/sourcemap-codec': 1.6.0 + math-intrinsics@1.1.0: {} media-typer@1.1.0: {} @@ -8249,7 +8209,7 @@ snapshots: object-treeify@1.1.33: {} - obug@2.1.2: {} + obug@2.2.1: {} on-finished@2.4.1: dependencies: @@ -8367,8 +8327,6 @@ snapshots: path-to-regexp@8.4.2: {} - pathe@2.0.3: {} - pg-cloudflare@1.4.0: optional: true @@ -8421,6 +8379,8 @@ snapshots: picomatch@4.0.4: {} + picomatch@4.0.7: {} + pkce-challenge@5.0.1: {} pluralize@8.0.0: {} @@ -8795,8 +8755,6 @@ snapshots: side-channel-map: 1.0.1 side-channel-weakmap: 1.0.2 - siginfo@2.0.0: {} - signal-exit@3.0.7: {} signal-exit@4.1.0: {} @@ -8813,8 +8771,6 @@ snapshots: sprintf-js@1.0.3: {} - stackback@0.0.2: {} - standardwebhooks@1.0.0: dependencies: '@stablelib/base64': 1.0.1 @@ -8822,7 +8778,7 @@ snapshots: statuses@2.0.2: {} - std-env@4.1.0: {} + std-env@4.3.0: {} stdin-discarder@0.2.2: {} @@ -8906,17 +8862,15 @@ snapshots: tiny-invariant@1.3.3: {} - tinybench@2.9.0: {} + tinybench@6.2.1: {} - tinyexec@1.2.4: {} + tinyexec@1.3.1: {} tinyglobby@0.2.17: dependencies: fdir: 6.5.0(picomatch@4.0.4) picomatch: 4.0.4 - tinyrainbow@3.1.0: {} - to-regex-range@5.0.1: dependencies: is-number: 7.0.0 @@ -9061,28 +9015,22 @@ snapshots: lightningcss: 1.32.0 tsx: 4.22.4 - vitest@4.1.8(@opentelemetry/api@1.9.1)(@types/node@22.19.20)(happy-dom@20.10.2)(vite@7.3.5(@types/node@22.19.20)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4)): - dependencies: - '@vitest/expect': 4.1.8 - '@vitest/mocker': 4.1.8(vite@7.3.5(@types/node@22.19.20)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4)) - '@vitest/pretty-format': 4.1.8 - '@vitest/runner': 4.1.8 - '@vitest/snapshot': 4.1.8 - '@vitest/spy': 4.1.8 - '@vitest/utils': 4.1.8 - es-module-lexer: 2.1.0 - expect-type: 1.3.0 - magic-string: 0.30.21 - obug: 2.1.2 - pathe: 2.0.3 - picomatch: 4.0.4 - std-env: 4.1.0 - tinybench: 2.9.0 - tinyexec: 1.2.4 + vitest@5.0.3(@opentelemetry/api@1.9.1)(@types/node@22.19.20)(happy-dom@20.10.2)(vite@7.3.5(@types/node@22.19.20)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4)): + dependencies: + '@types/chai': 5.2.3 + '@vitest/mocker': 5.0.3(vite@7.3.5(@types/node@22.19.20)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4)) + chai: 6.2.2 + es-module-lexer: 2.3.2 + expect-type: 1.4.0 + magic-string: 1.4.3 + obug: 2.2.1 + picomatch: 4.0.7 + std-env: 4.3.0 + tinybench: 6.2.1 + tinyexec: 1.3.1 tinyglobby: 0.2.17 - tinyrainbow: 3.1.0 vite: 7.3.5(@types/node@22.19.20)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4) - why-is-node-running: 2.3.0 + why-is-node-running: 3.2.1 optionalDependencies: '@opentelemetry/api': 1.9.1 '@types/node': 22.19.20 @@ -9090,28 +9038,22 @@ snapshots: transitivePeerDependencies: - msw - vitest@4.1.8(@opentelemetry/api@1.9.1)(@types/node@25.9.3)(happy-dom@20.10.2)(vite@7.3.5(@types/node@25.9.3)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4)): - dependencies: - '@vitest/expect': 4.1.8 - '@vitest/mocker': 4.1.8(vite@7.3.5(@types/node@25.9.3)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4)) - '@vitest/pretty-format': 4.1.8 - '@vitest/runner': 4.1.8 - '@vitest/snapshot': 4.1.8 - '@vitest/spy': 4.1.8 - '@vitest/utils': 4.1.8 - es-module-lexer: 2.1.0 - expect-type: 1.3.0 - magic-string: 0.30.21 - obug: 2.1.2 - pathe: 2.0.3 - picomatch: 4.0.4 - std-env: 4.1.0 - tinybench: 2.9.0 - tinyexec: 1.2.4 + vitest@5.0.3(@opentelemetry/api@1.9.1)(@types/node@25.9.3)(happy-dom@20.10.2)(vite@7.3.5(@types/node@25.9.3)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4)): + dependencies: + '@types/chai': 5.2.3 + '@vitest/mocker': 5.0.3(vite@7.3.5(@types/node@25.9.3)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4)) + chai: 6.2.2 + es-module-lexer: 2.3.2 + expect-type: 1.4.0 + magic-string: 1.4.3 + obug: 2.2.1 + picomatch: 4.0.7 + std-env: 4.3.0 + tinybench: 6.2.1 + tinyexec: 1.3.1 tinyglobby: 0.2.17 - tinyrainbow: 3.1.0 vite: 7.3.5(@types/node@25.9.3)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4) - why-is-node-running: 2.3.0 + why-is-node-running: 3.2.1 optionalDependencies: '@opentelemetry/api': 1.9.1 '@types/node': 25.9.3 @@ -9133,10 +9075,7 @@ snapshots: dependencies: isexe: 3.1.5 - why-is-node-running@2.3.0: - dependencies: - siginfo: 2.0.0 - stackback: 0.0.2 + why-is-node-running@3.2.1: {} word-wrap@1.2.5: {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 2e77c2f2..2b623a1c 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -21,5 +21,5 @@ catalog: ai: ^6.0.174 typescript: ^5.9.3 vite: ^7.3.1 - vitest: ^4.1.5 + vitest: ^5.0.3 zod: ^4.4.3 From 1129e4349619a1e42061a573c3844811f4679cec Mon Sep 17 00:00:00 2001 From: Matt Rossman <22670878+mattrossman@users.noreply.github.com> Date: Wed, 7 Oct 2026 13:48:50 -0400 Subject: [PATCH 10/33] chore: temporary eval change for refresh pilot --- .../resolve-reliability-001-unhealthy-project-recovery/EVAL.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/evals/regression/resolve-reliability-001-unhealthy-project-recovery/EVAL.ts b/evals/regression/resolve-reliability-001-unhealthy-project-recovery/EVAL.ts index d1382da7..d9c492ac 100644 --- a/evals/regression/resolve-reliability-001-unhealthy-project-recovery/EVAL.ts +++ b/evals/regression/resolve-reliability-001-unhealthy-project-recovery/EVAL.ts @@ -35,3 +35,5 @@ const scorer: ToolScorer = async (ctx) => { }; export default scorer; + +// Temporary change to exercise the refresh results commit. From e39410fff7db246241923ddedc753bc945f48f3b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 7 Oct 2026 17:53:10 +0000 Subject: [PATCH 11/33] chore: refresh eval results --- .../web/src/data/regression-eval-results.json | 458 ++++++++---------- 1 file changed, 210 insertions(+), 248 deletions(-) diff --git a/apps/web/src/data/regression-eval-results.json b/apps/web/src/data/regression-eval-results.json index ce73398e..2d1fc861 100644 --- a/apps/web/src/data/regression-eval-results.json +++ b/apps/web/src/data/regression-eval-results.json @@ -4511,7 +4511,7 @@ { "name": "answered unhealthy project recovery question safely", "passed": true, - "judgeNotes": "Recommends restarting the database rather than pausing and restoring the project, cites Supabase’s unhealthy-services guidance, and identifies resource or tuning issues to investigate if the problem returns. It could be stronger on logs, scaling, and contacting support." + "judgeNotes": "Recommends restarting the database first, distinguishes restart from pause/restore, and advises contacting support if a restart is unavailable. It does not present restore as a fix for unhealthy services." } ], "skills": { @@ -4527,90 +4527,135 @@ "docs": { "calls": [ { - "source": "search_docs", - "query": "query { searchDocs(query: \"Monitoring and Debugging Supabase project unhealthy dashboard\", limit: 5) { nodes { title href content } }", - "hasContent": true, + "source": "web_search", + "query": "site:supabase.com/docs project unhealthy restart pause restore project dashboard restart project", "pages": [] }, { - "source": "shell_fetch", - "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -c 5000'", + "source": "search_docs", + "query": "query { searchDocs(query: \"Project Status reports unhealthy services restart database\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/changelog.md" + "url": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services", + "title": "Project Status reports unhealthy services" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/failed-to-run-sql-query-connection-terminated-due-to-connection-timeout", + "title": "Error: Failed to run sql query: Connection terminated due to connection timeout" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17", + "title": "Upgrade to Postgres 17" + }, + { + "url": "https://supabase.com/docs/guides/database/replication/pipelines-faq", + "title": "Pipelines FAQ" + }, + { + "url": "https://supabase.com/docs/guides/database/prisma/prisma-troubleshooting", + "title": "Troubleshooting prisma errors" } ], - "resultChars": 5003 + "resultChars": 48330 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"restart project pause restore unhealthy dashboard Supabase project\", limit: 6) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"pause restore project Free plan restore project restart\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", - "title": "Restoring a downloaded backup locally" - }, - { - "url": "https://supabase.com/docs/guides/platform/free-project-pausing", - "title": "Project Pausing" + "url": "https://supabase.com/docs/guides/platform/webhooks/events", + "title": "Platform Webhook Events" }, { "url": "https://supabase.com/docs/guides/platform/upgrading", "title": "Upgrading" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/restore-project-after-90-days-pause", - "title": "How To Restore a Project Paused for More Than 1 Year" + "url": "https://supabase.com/docs/guides/platform/free-project-pausing", + "title": "Project Pausing" }, { "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres", "title": "Migrate from Postgres to Supabase" }, { - "url": "https://supabase.com/docs/guides/platform/migrating-within-supabase/dashboard-restore", - "title": "Restore Dashboard backup" + "url": "https://supabase.com/docs/guides/platform/migrating-within-supabase", + "title": "Migrating within Supabase" } ], - "resultChars": 53197 + "resultChars": 53231 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Monitoring and Debugging guide troubleshoot project health status\", limit: 5) { nodes { title href content } }", + "query": "query { searchDocs(query: \"cooldown period pause and restore new disk project\", limit: 5) { nodes { title href content } } }", "hasContent": true, - "pages": [] + "pages": [ + { + "url": "https://supabase.com/docs/guides/troubleshooting/how-to-bypass-cooldown-period", + "title": "How to bypass cooldown period" + }, + { + "url": "https://supabase.com/docs/guides/platform/upgrading", + "title": "Upgrading" + }, + { + "url": "https://supabase.com/docs/guides/platform/free-project-pausing", + "title": "Project Pausing" + }, + { + "url": "https://supabase.com/docs/guides/platform/delete-project", + "title": "Deleting Your Project" + }, + { + "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", + "title": "Restoring a downloaded backup locally" + } + ], + "resultChars": 24769 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Restarting a Supabase project from dashboard\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Monitoring and Debugging Supabase unhealthy database troubleshooting\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/ai/examples/nextjs-vector-search", - "title": "Vector search with Next.js and OpenAI" + "url": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services", + "title": "Project Status reports unhealthy services" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker", - "title": "Self-Hosting with Docker" + "url": "https://supabase.com/docs/guides/troubleshooting/supabase-grafana-memory-charts", + "title": "Interpreting Supabase Grafana Memory Charts" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", - "title": "Build a User Management App with Next.js" + "url": "https://supabase.com/docs/guides/observability", + "title": "Observability" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nuxt-3", - "title": "Build a User Management App with Nuxt 3" + "url": "https://supabase.com/docs/guides/database/postgres/timeouts", + "title": "Timeouts" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/nxdomain-error-connecting-to-a-supabase-project" + "url": "https://supabase.com/docs/guides/deployment/shared-responsibility-model", + "title": "Shared Responsibility Model" } ], - "resultChars": 117041 + "resultChars": 21591 + }, + { + "source": "web_search", + "query": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services" + } + ] }, { "source": "web_search", - "query": "site:supabase.com/docs project restart dashboard restart project unhealthy", + "query": "Supabase status current incident dashboard October 7 2026", "pages": [] } ] @@ -4618,22 +4663,22 @@ "usage": [ { "model": "gpt-6-luna", - "inputTokens": 81606, - "cacheReadInputTokens": 38199, - "cacheWriteInputTokens": 36992, - "outputTokens": 1461 + "inputTokens": 75907, + "cacheReadInputTokens": 16867, + "cacheWriteInputTokens": 41085, + "outputTokens": 1818 } ], - "stepCount": 4, - "toolCallCount": 7, - "agentRunDurationMs": 28765, + "stepCount": 3, + "toolCallCount": 8, + "agentRunDurationMs": 37160, "sandboxUsage": { - "activeCpuDurationMs": 120458, - "duration": 124157, + "activeCpuDurationMs": 128220, + "duration": 133751, "memory": 8192, "networkTransfer": { - "ingress": 765306400, - "egress": 3940667 + "ingress": 765373908, + "egress": 4459733 } }, "prompt": "My Supabase dashboard says my project is unhealthy, and the dashboard is unusable.\n\nWould restart or pause/restore be better?", @@ -4665,7 +4710,7 @@ { "name": "answered unhealthy project recovery question safely", "passed": true, - "judgeNotes": "Rejects pause/restore for an active unhealthy project and recommends a database restart as a concrete first recovery step, with support escalation if the dashboard remains unusable. It does not conflate restart with restore." + "judgeNotes": "Recommends restarting first, distinguishes restart from restore, warns against pause/restore as an unhealthy-project fix, and points to troubleshooting guidance and support." } ], "skills": { @@ -4674,136 +4719,42 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase", - "supabase-postgres-best-practices" + "supabase" ] }, "docs": { "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"restore a paused project restart project Supabase dashboard pause project\", limit: 5) { nodes { ... on Guide { title href content } } }", - "hasContent": true, - "pages": [] - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -c 12000'", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 11955 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Monitoring and Debugging Supabase project unhealthy restart project pause restore project dashboard\", limit: 5) { nodes { ... on Guide { title href content } } } }", + "query": "query { searchDocs(query: \"restart pause restore unhealthy project recovery project health Supabase dashboard\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", "title": "Restoring a downloaded backup locally" }, - { - "url": "https://supabase.com/docs/guides/platform/free-project-pausing", - "title": "Project Pausing" - }, { "url": "https://supabase.com/docs/guides/platform/upgrading", "title": "Upgrading" }, { - "url": "https://supabase.com/docs/guides/observability", - "title": "Observability" - } - ], - "resultChars": 17891 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/docs/guides/monitoring-and-debugging.md | head -c 8000'", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging.md" - } - ], - "resultChars": 3394 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Monitoring and Debugging\", limit: 5) { nodes { ... on Guide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/database/postgres/data-deletion", - "title": "Deleting data and dropping objects safely" - }, - { - "url": "https://supabase.com/docs/guides/observability/reports", - "title": "Reports" - }, - { - "url": "https://supabase.com/docs/guides/database/replication/pipelines-monitoring", - "title": "Monitor pipeline status" - } - ], - "resultChars": 64536 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"project is unhealthy dashboard inaccessible support\", limit: 5) { nodes { ... on Guide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/platform/temporary-access", - "title": "Temporary access" - }, - { - "url": "https://supabase.com/docs/guides/platform/network-restrictions", - "title": "Network Restrictions" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" - } - ], - "resultChars": 25516 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"restart project\", limit: 5) { nodes { ... on Guide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/database/replication/pipelines-faq", - "title": "Pipelines FAQ" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" + "url": "https://supabase.com/docs/guides/platform/free-project-pausing", + "title": "Project Pausing" }, { - "url": "https://supabase.com/docs/guides/platform/webhooks/events", - "title": "Platform Webhook Events" + "url": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services", + "title": "Project Status reports unhealthy services" }, { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" + "url": "https://supabase.com/docs/guides/troubleshooting/restore-project-after-90-days-pause", + "title": "How To Restore a Project Paused for More Than 1 Year" } ], - "resultChars": 70609 + "resultChars": 22404 }, { "source": "web_search", - "query": "site:supabase.com/docs/guides/platform restart project restart Supabase project dashboard", - "pages": [] - }, - { - "source": "web_search", - "query": "Supabase status current incidents October 7 2026 dashboard unhealthy", + "query": "site:supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services restart database unhealthy Supabase", "pages": [] } ] @@ -4811,22 +4762,22 @@ "usage": [ { "model": "gpt-6-luna", - "inputTokens": 98253, - "cacheReadInputTokens": 39570, - "cacheWriteInputTokens": 43666, - "outputTokens": 2113 + "inputTokens": 53154, + "cacheReadInputTokens": 16860, + "cacheWriteInputTokens": 31969, + "outputTokens": 939 } ], - "stepCount": 4, - "toolCallCount": 10, - "agentRunDurationMs": 34607, + "stepCount": 3, + "toolCallCount": 3, + "agentRunDurationMs": 23169, "sandboxUsage": { - "activeCpuDurationMs": 125130, - "duration": 129819, + "activeCpuDurationMs": 167046, + "duration": 146858, "memory": 8192, "networkTransfer": { - "ingress": 764645803, - "egress": 3709271 + "ingress": 762280782, + "egress": 2110911 } }, "prompt": "My Supabase dashboard says my project is unhealthy, and the dashboard is unusable.\n\nWould restart or pause/restore be better?", @@ -4858,7 +4809,7 @@ { "name": "answered unhealthy project recovery question safely", "passed": true, - "judgeNotes": "Recommends restarting first, distinguishes restart from backup restore, warns against pause/restore as a workaround, and suggests checking resource usage or contacting support." + "judgeNotes": "Recommends a database restart as the first step, warns against pausing and restoring to troubleshoot, and points to resource checks, scaling, troubleshooting guidance, and support. It could more clearly explain that restore is for backup/data recovery." } ], "skills": { @@ -4867,54 +4818,54 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase", - "supabase-postgres-best-practices" + "supabase" ] }, "docs": { "calls": [ - { - "source": "web_search", - "query": "site:supabase.com/docs project unhealthy restart pause restore Supabase project", - "pages": [] - }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Project Status reports unhealthy services restart database dashboard project settings\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Supabase project status unhealthy restart project dashboard\", limit: 8) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", "hasContent": true, "pages": [ { "url": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services", "title": "Project Status reports unhealthy services" }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/supabase-dashboard-not-loading-project-not-loading-on-dashboard-LfMq9F", + "title": "Supabase dashboard not loading/ Project not loading on dashboard" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/why-do-i-see-auth--api-requests-in-the-dashboard-my-app-has-no-users-CyadiO", + "title": "Why do I see Auth & API requests in the dashboard? My app has no users" + }, { "url": "https://supabase.com/docs/guides/troubleshooting/failed-to-run-sql-query-connection-terminated-due-to-connection-timeout", "title": "Error: Failed to run sql query: Connection terminated due to connection timeout" }, { - "url": "https://supabase.com/docs/guides/platform/temporary-access", - "title": "Temporary access" + "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", + "title": "Restoring a downloaded backup locally" }, { - "url": "https://supabase.com/docs/guides/platform/read-replicas", - "title": "Read Replicas" + "url": "https://supabase.com/docs/guides/database/postgres/timeouts", + "title": "Timeouts" }, { - "url": "https://supabase.com/docs/guides/database/replication/pipelines-faq", - "title": "Pipelines FAQ" + "url": "https://supabase.com/docs/guides/platform/free-project-pausing", + "title": "Project Pausing" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/remove-superuser-access", + "title": "Remove Superuser Access from Studio" } ], - "resultChars": 36932 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Monitoring and Debugging Supabase dashboard unavailable project unhealthy\", limit: 5) { nodes { title href content } }", - "hasContent": true, - "pages": [] + "resultChars": 19697 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"pause restore project data backups restart Supabase project services\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"restart project unhealthy pause restore project dashboard unavailable Supabase troubleshooting\", limit: 8) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", "hasContent": true, "pages": [ { @@ -4926,46 +4877,53 @@ "title": "Upgrading" }, { - "url": "https://supabase.com/docs/guides/platform/personal-access-tokens", - "title": "Personal Access Tokens" + "url": "https://supabase.com/docs/guides/platform/free-project-pausing", + "title": "Project Pausing" }, { - "url": "https://supabase.com/docs/guides/platform/access-control", - "title": "Access Control" + "url": "https://supabase.com/docs/guides/troubleshooting/restore-project-after-90-days-pause", + "title": "How To Restore a Project Paused for More Than 1 Year" }, { - "url": "https://supabase.com/docs/guides/platform/free-project-pausing", - "title": "Project Pausing" + "url": "https://supabase.com/docs/guides/platform/migrating-within-supabase/dashboard-restore", + "title": "Restore Dashboard backup" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/supabase-dashboard-not-loading-project-not-loading-on-dashboard-LfMq9F", + "title": "Supabase dashboard not loading/ Project not loading on dashboard" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/http-api-issues", + "title": "Diagnose HTTP API issues" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres", + "title": "Migrate from Postgres to Supabase" } ], - "resultChars": 133827 - }, - { - "source": "web_search", - "query": "Supabase status status.supabase.com incident dashboard outage October 7 2026", - "pages": [] + "resultChars": 58169 } ] }, "usage": [ { "model": "gpt-6-luna", - "inputTokens": 87918, - "cacheReadInputTokens": 36684, - "cacheWriteInputTokens": 35039, - "outputTokens": 1543 + "inputTokens": 52059, + "cacheReadInputTokens": 16861, + "cacheWriteInputTokens": 34994, + "outputTokens": 1007 } ], - "stepCount": 4, - "toolCallCount": 7, - "agentRunDurationMs": 38739, + "stepCount": 3, + "toolCallCount": 3, + "agentRunDurationMs": 22730, "sandboxUsage": { - "activeCpuDurationMs": 130077, - "duration": 135984, + "activeCpuDurationMs": 128111, + "duration": 123994, "memory": 8192, "networkTransfer": { - "ingress": 764881831, - "egress": 3406252 + "ingress": 765166920, + "egress": 4119857 } }, "prompt": "My Supabase dashboard says my project is unhealthy, and the dashboard is unusable.\n\nWould restart or pause/restore be better?", @@ -9728,12 +9686,12 @@ ], "suite": "regression", "interface": "mcp", - "passed": true, + "passed": false, "checks": [ { "name": "answered unhealthy project recovery question safely", - "passed": true, - "judgeNotes": "Recommends restarting as the first recovery step, rejects pause-and-restore for an unhealthy project, and provides concrete checks and a support escalation path." + "passed": false, + "judgeNotes": "It correctly warns against pause/restore and identifies restart as a recovery attempt, but does not explain that restore is backup/data recovery or provide enough concrete unhealthy-project troubleshooting guidance, such as checking logs and resource pressure or contacting support if the issue persists." } ], "skills": { @@ -9741,33 +9699,27 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "web_search", - "query": "site:supabase.com/docs project unhealthy restart pause restore dashboard project", - "pages": [] - } - ] + "calls": [] }, "usage": [ { "model": "gpt-6-luna", - "inputTokens": 20317, + "inputTokens": 11916, "cacheReadInputTokens": 0, - "cacheWriteInputTokens": 11850, - "outputTokens": 646 + "cacheWriteInputTokens": 11848, + "outputTokens": 185 } ], "stepCount": 1, - "toolCallCount": 1, - "agentRunDurationMs": 14625, + "toolCallCount": 0, + "agentRunDurationMs": 10969, "sandboxUsage": { - "activeCpuDurationMs": 118702, - "duration": 108012, + "activeCpuDurationMs": 165530, + "duration": 133851, "memory": 8192, "networkTransfer": { - "ingress": 764179303, - "egress": 2911081 + "ingress": 762065864, + "egress": 1758995 } }, "prompt": "My Supabase dashboard says my project is unhealthy, and the dashboard is unusable.\n\nWould restart or pause/restore be better?", @@ -9799,7 +9751,7 @@ { "name": "answered unhealthy project recovery question safely", "passed": true, - "judgeNotes": "Recommends restarting first, distinguishes restart from backup restore, and advises checking resource usage and contacting support if the issue persists." + "judgeNotes": "Recommends restarting first, rejects pause-and-restore as the initial recovery path, and advises checking workload and contacting support if the problem persists." } ], "skills": { @@ -9810,7 +9762,7 @@ "calls": [ { "source": "web_search", - "query": "site:supabase.com/docs project unhealthy restart pause restore project dashboard", + "query": "site:supabase.com/docs project unhealthy restart pause restore dashboard unusable", "pages": [] } ] @@ -9818,22 +9770,22 @@ "usage": [ { "model": "gpt-6-luna", - "inputTokens": 20204, + "inputTokens": 19975, "cacheReadInputTokens": 0, - "cacheWriteInputTokens": 11854, - "outputTokens": 944 + "cacheWriteInputTokens": 11850, + "outputTokens": 496 } ], "stepCount": 1, "toolCallCount": 1, - "agentRunDurationMs": 17377, + "agentRunDurationMs": 17660, "sandboxUsage": { - "activeCpuDurationMs": 156866, - "duration": 133050, + "activeCpuDurationMs": 123354, + "duration": 115758, "memory": 8192, "networkTransfer": { - "ingress": 761651066, - "egress": 1501409 + "ingress": 764883428, + "egress": 3556376 } }, "prompt": "My Supabase dashboard says my project is unhealthy, and the dashboard is unusable.\n\nWould restart or pause/restore be better?", @@ -9860,12 +9812,12 @@ ], "suite": "regression", "interface": "mcp", - "passed": true, + "passed": false, "checks": [ { "name": "answered unhealthy project recovery question safely", - "passed": true, - "judgeNotes": "Recommends restarting first, warns against pause/restore as a fix, and suggests checking resource usage and contacting support. It could more clearly identify restore as backup recovery, but does not conflate it with restart." + "passed": false, + "judgeNotes": "The answer gives a safe restart-first step and rejects pause/restore for an unhealthy project, but it describes restore as resuming a paused project rather than distinguishing backup/data restore from restart as the rubric requires." } ], "skills": { @@ -9876,30 +9828,40 @@ "calls": [ { "source": "web_search", - "query": "site:supabase.com/docs project unhealthy restart pause restore Supabase dashboard", + "query": "site:supabase.com/docs project unhealthy restart pause restore Supabase project", "pages": [] + }, + { + "source": "web_search", + "query": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services" + } + ] } ] }, "usage": [ { "model": "gpt-6-luna", - "inputTokens": 20127, + "inputTokens": 29452, "cacheReadInputTokens": 0, "cacheWriteInputTokens": 11848, - "outputTokens": 701 + "outputTokens": 1072 } ], "stepCount": 1, - "toolCallCount": 1, - "agentRunDurationMs": 15254, + "toolCallCount": 2, + "agentRunDurationMs": 25698, "sandboxUsage": { - "activeCpuDurationMs": 170947, - "duration": 142633, + "activeCpuDurationMs": 123509, + "duration": 126694, "memory": 8192, "networkTransfer": { - "ingress": 761594460, - "egress": 1426068 + "ingress": 764773255, + "egress": 3763194 } }, "prompt": "My Supabase dashboard says my project is unhealthy, and the dashboard is unusable.\n\nWould restart or pause/restore be better?", From dbd7afd5b2c52be595979dae154614e067142bd6 Mon Sep 17 00:00:00 2001 From: Matt Rossman <22670878+mattrossman@users.noreply.github.com> Date: Wed, 7 Oct 2026 13:55:42 -0400 Subject: [PATCH 12/33] Revert "chore: refresh eval results" This reverts commit e39410fff7db246241923ddedc753bc945f48f3b. --- .../web/src/data/regression-eval-results.json | 458 ++++++++++-------- 1 file changed, 248 insertions(+), 210 deletions(-) diff --git a/apps/web/src/data/regression-eval-results.json b/apps/web/src/data/regression-eval-results.json index 2d1fc861..ce73398e 100644 --- a/apps/web/src/data/regression-eval-results.json +++ b/apps/web/src/data/regression-eval-results.json @@ -4511,7 +4511,7 @@ { "name": "answered unhealthy project recovery question safely", "passed": true, - "judgeNotes": "Recommends restarting the database first, distinguishes restart from pause/restore, and advises contacting support if a restart is unavailable. It does not present restore as a fix for unhealthy services." + "judgeNotes": "Recommends restarting the database rather than pausing and restoring the project, cites Supabase’s unhealthy-services guidance, and identifies resource or tuning issues to investigate if the problem returns. It could be stronger on logs, scaling, and contacting support." } ], "skills": { @@ -4527,135 +4527,90 @@ "docs": { "calls": [ { - "source": "web_search", - "query": "site:supabase.com/docs project unhealthy restart pause restore project dashboard restart project", + "source": "search_docs", + "query": "query { searchDocs(query: \"Monitoring and Debugging Supabase project unhealthy dashboard\", limit: 5) { nodes { title href content } }", + "hasContent": true, "pages": [] }, { - "source": "search_docs", - "query": "query { searchDocs(query: \"Project Status reports unhealthy services restart database\", limit: 5) { nodes { title href content } } }", + "source": "shell_fetch", + "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -c 5000'", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services", - "title": "Project Status reports unhealthy services" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/failed-to-run-sql-query-connection-terminated-due-to-connection-timeout", - "title": "Error: Failed to run sql query: Connection terminated due to connection timeout" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17", - "title": "Upgrade to Postgres 17" - }, - { - "url": "https://supabase.com/docs/guides/database/replication/pipelines-faq", - "title": "Pipelines FAQ" - }, - { - "url": "https://supabase.com/docs/guides/database/prisma/prisma-troubleshooting", - "title": "Troubleshooting prisma errors" + "url": "https://supabase.com/changelog.md" } ], - "resultChars": 48330 + "resultChars": 5003 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"pause restore project Free plan restore project restart\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"restart project pause restore unhealthy dashboard Supabase project\", limit: 6) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/platform/webhooks/events", - "title": "Platform Webhook Events" + "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", + "title": "Restoring a downloaded backup locally" + }, + { + "url": "https://supabase.com/docs/guides/platform/free-project-pausing", + "title": "Project Pausing" }, { "url": "https://supabase.com/docs/guides/platform/upgrading", "title": "Upgrading" }, { - "url": "https://supabase.com/docs/guides/platform/free-project-pausing", - "title": "Project Pausing" + "url": "https://supabase.com/docs/guides/troubleshooting/restore-project-after-90-days-pause", + "title": "How To Restore a Project Paused for More Than 1 Year" }, { "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres", "title": "Migrate from Postgres to Supabase" }, { - "url": "https://supabase.com/docs/guides/platform/migrating-within-supabase", - "title": "Migrating within Supabase" + "url": "https://supabase.com/docs/guides/platform/migrating-within-supabase/dashboard-restore", + "title": "Restore Dashboard backup" } ], - "resultChars": 53231 + "resultChars": 53197 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"cooldown period pause and restore new disk project\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Monitoring and Debugging guide troubleshoot project health status\", limit: 5) { nodes { title href content } }", "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/troubleshooting/how-to-bypass-cooldown-period", - "title": "How to bypass cooldown period" - }, - { - "url": "https://supabase.com/docs/guides/platform/upgrading", - "title": "Upgrading" - }, - { - "url": "https://supabase.com/docs/guides/platform/free-project-pausing", - "title": "Project Pausing" - }, - { - "url": "https://supabase.com/docs/guides/platform/delete-project", - "title": "Deleting Your Project" - }, - { - "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", - "title": "Restoring a downloaded backup locally" - } - ], - "resultChars": 24769 + "pages": [] }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Monitoring and Debugging Supabase unhealthy database troubleshooting\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Restarting a Supabase project from dashboard\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services", - "title": "Project Status reports unhealthy services" + "url": "https://supabase.com/docs/guides/ai/examples/nextjs-vector-search", + "title": "Vector search with Next.js and OpenAI" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/supabase-grafana-memory-charts", - "title": "Interpreting Supabase Grafana Memory Charts" + "url": "https://supabase.com/docs/guides/self-hosting/docker", + "title": "Self-Hosting with Docker" }, { - "url": "https://supabase.com/docs/guides/observability", - "title": "Observability" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", + "title": "Build a User Management App with Next.js" }, { - "url": "https://supabase.com/docs/guides/database/postgres/timeouts", - "title": "Timeouts" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nuxt-3", + "title": "Build a User Management App with Nuxt 3" }, { - "url": "https://supabase.com/docs/guides/deployment/shared-responsibility-model", - "title": "Shared Responsibility Model" + "url": "https://supabase.com/docs/guides/troubleshooting/nxdomain-error-connecting-to-a-supabase-project" } ], - "resultChars": 21591 - }, - { - "source": "web_search", - "query": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services" - } - ] + "resultChars": 117041 }, { "source": "web_search", - "query": "Supabase status current incident dashboard October 7 2026", + "query": "site:supabase.com/docs project restart dashboard restart project unhealthy", "pages": [] } ] @@ -4663,22 +4618,22 @@ "usage": [ { "model": "gpt-6-luna", - "inputTokens": 75907, - "cacheReadInputTokens": 16867, - "cacheWriteInputTokens": 41085, - "outputTokens": 1818 + "inputTokens": 81606, + "cacheReadInputTokens": 38199, + "cacheWriteInputTokens": 36992, + "outputTokens": 1461 } ], - "stepCount": 3, - "toolCallCount": 8, - "agentRunDurationMs": 37160, + "stepCount": 4, + "toolCallCount": 7, + "agentRunDurationMs": 28765, "sandboxUsage": { - "activeCpuDurationMs": 128220, - "duration": 133751, + "activeCpuDurationMs": 120458, + "duration": 124157, "memory": 8192, "networkTransfer": { - "ingress": 765373908, - "egress": 4459733 + "ingress": 765306400, + "egress": 3940667 } }, "prompt": "My Supabase dashboard says my project is unhealthy, and the dashboard is unusable.\n\nWould restart or pause/restore be better?", @@ -4710,7 +4665,7 @@ { "name": "answered unhealthy project recovery question safely", "passed": true, - "judgeNotes": "Recommends restarting first, distinguishes restart from restore, warns against pause/restore as an unhealthy-project fix, and points to troubleshooting guidance and support." + "judgeNotes": "Rejects pause/restore for an active unhealthy project and recommends a database restart as a concrete first recovery step, with support escalation if the dashboard remains unusable. It does not conflate restart with restore." } ], "skills": { @@ -4719,42 +4674,136 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase" + "supabase", + "supabase-postgres-best-practices" ] }, "docs": { "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"restart pause restore unhealthy project recovery project health Supabase dashboard\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"restore a paused project restart project Supabase dashboard pause project\", limit: 5) { nodes { ... on Guide { title href content } } }", + "hasContent": true, + "pages": [] + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -c 12000'", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 11955 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Monitoring and Debugging Supabase project unhealthy restart project pause restore project dashboard\", limit: 5) { nodes { ... on Guide { title href content } } } }", "hasContent": true, "pages": [ { "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", "title": "Restoring a downloaded backup locally" }, + { + "url": "https://supabase.com/docs/guides/platform/free-project-pausing", + "title": "Project Pausing" + }, { "url": "https://supabase.com/docs/guides/platform/upgrading", "title": "Upgrading" }, { - "url": "https://supabase.com/docs/guides/platform/free-project-pausing", - "title": "Project Pausing" + "url": "https://supabase.com/docs/guides/observability", + "title": "Observability" + } + ], + "resultChars": 17891 + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/docs/guides/monitoring-and-debugging.md | head -c 8000'", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging.md" + } + ], + "resultChars": 3394 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Monitoring and Debugging\", limit: 5) { nodes { ... on Guide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/postgres/data-deletion", + "title": "Deleting data and dropping objects safely" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services", - "title": "Project Status reports unhealthy services" + "url": "https://supabase.com/docs/guides/observability/reports", + "title": "Reports" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/restore-project-after-90-days-pause", - "title": "How To Restore a Project Paused for More Than 1 Year" + "url": "https://supabase.com/docs/guides/database/replication/pipelines-monitoring", + "title": "Monitor pipeline status" + } + ], + "resultChars": 64536 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"project is unhealthy dashboard inaccessible support\", limit: 5) { nodes { ... on Guide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/platform/temporary-access", + "title": "Temporary access" + }, + { + "url": "https://supabase.com/docs/guides/platform/network-restrictions", + "title": "Network Restrictions" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" } ], - "resultChars": 22404 + "resultChars": 25516 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"restart project\", limit: 5) { nodes { ... on Guide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/replication/pipelines-faq", + "title": "Pipelines FAQ" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows", + "title": "Local development workflow" + }, + { + "url": "https://supabase.com/docs/guides/platform/webhooks/events", + "title": "Platform Webhook Events" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" + } + ], + "resultChars": 70609 }, { "source": "web_search", - "query": "site:supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services restart database unhealthy Supabase", + "query": "site:supabase.com/docs/guides/platform restart project restart Supabase project dashboard", + "pages": [] + }, + { + "source": "web_search", + "query": "Supabase status current incidents October 7 2026 dashboard unhealthy", "pages": [] } ] @@ -4762,22 +4811,22 @@ "usage": [ { "model": "gpt-6-luna", - "inputTokens": 53154, - "cacheReadInputTokens": 16860, - "cacheWriteInputTokens": 31969, - "outputTokens": 939 + "inputTokens": 98253, + "cacheReadInputTokens": 39570, + "cacheWriteInputTokens": 43666, + "outputTokens": 2113 } ], - "stepCount": 3, - "toolCallCount": 3, - "agentRunDurationMs": 23169, + "stepCount": 4, + "toolCallCount": 10, + "agentRunDurationMs": 34607, "sandboxUsage": { - "activeCpuDurationMs": 167046, - "duration": 146858, + "activeCpuDurationMs": 125130, + "duration": 129819, "memory": 8192, "networkTransfer": { - "ingress": 762280782, - "egress": 2110911 + "ingress": 764645803, + "egress": 3709271 } }, "prompt": "My Supabase dashboard says my project is unhealthy, and the dashboard is unusable.\n\nWould restart or pause/restore be better?", @@ -4809,7 +4858,7 @@ { "name": "answered unhealthy project recovery question safely", "passed": true, - "judgeNotes": "Recommends a database restart as the first step, warns against pausing and restoring to troubleshoot, and points to resource checks, scaling, troubleshooting guidance, and support. It could more clearly explain that restore is for backup/data recovery." + "judgeNotes": "Recommends restarting first, distinguishes restart from backup restore, warns against pause/restore as a workaround, and suggests checking resource usage or contacting support." } ], "skills": { @@ -4818,54 +4867,54 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase" + "supabase", + "supabase-postgres-best-practices" ] }, "docs": { "calls": [ + { + "source": "web_search", + "query": "site:supabase.com/docs project unhealthy restart pause restore Supabase project", + "pages": [] + }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase project status unhealthy restart project dashboard\", limit: 8) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", + "query": "query { searchDocs(query: \"Project Status reports unhealthy services restart database dashboard project settings\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { "url": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services", "title": "Project Status reports unhealthy services" }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/supabase-dashboard-not-loading-project-not-loading-on-dashboard-LfMq9F", - "title": "Supabase dashboard not loading/ Project not loading on dashboard" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/why-do-i-see-auth--api-requests-in-the-dashboard-my-app-has-no-users-CyadiO", - "title": "Why do I see Auth & API requests in the dashboard? My app has no users" - }, { "url": "https://supabase.com/docs/guides/troubleshooting/failed-to-run-sql-query-connection-terminated-due-to-connection-timeout", "title": "Error: Failed to run sql query: Connection terminated due to connection timeout" }, { - "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", - "title": "Restoring a downloaded backup locally" + "url": "https://supabase.com/docs/guides/platform/temporary-access", + "title": "Temporary access" }, { - "url": "https://supabase.com/docs/guides/database/postgres/timeouts", - "title": "Timeouts" - }, - { - "url": "https://supabase.com/docs/guides/platform/free-project-pausing", - "title": "Project Pausing" + "url": "https://supabase.com/docs/guides/platform/read-replicas", + "title": "Read Replicas" }, { - "url": "https://supabase.com/docs/guides/self-hosting/remove-superuser-access", - "title": "Remove Superuser Access from Studio" + "url": "https://supabase.com/docs/guides/database/replication/pipelines-faq", + "title": "Pipelines FAQ" } ], - "resultChars": 19697 + "resultChars": 36932 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Monitoring and Debugging Supabase dashboard unavailable project unhealthy\", limit: 5) { nodes { title href content } }", + "hasContent": true, + "pages": [] }, { "source": "search_docs", - "query": "query { searchDocs(query: \"restart project unhealthy pause restore project dashboard unavailable Supabase troubleshooting\", limit: 8) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", + "query": "query { searchDocs(query: \"pause restore project data backups restart Supabase project services\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { @@ -4877,53 +4926,46 @@ "title": "Upgrading" }, { - "url": "https://supabase.com/docs/guides/platform/free-project-pausing", - "title": "Project Pausing" + "url": "https://supabase.com/docs/guides/platform/personal-access-tokens", + "title": "Personal Access Tokens" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/restore-project-after-90-days-pause", - "title": "How To Restore a Project Paused for More Than 1 Year" + "url": "https://supabase.com/docs/guides/platform/access-control", + "title": "Access Control" }, { - "url": "https://supabase.com/docs/guides/platform/migrating-within-supabase/dashboard-restore", - "title": "Restore Dashboard backup" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/supabase-dashboard-not-loading-project-not-loading-on-dashboard-LfMq9F", - "title": "Supabase dashboard not loading/ Project not loading on dashboard" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/http-api-issues", - "title": "Diagnose HTTP API issues" - }, - { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres", - "title": "Migrate from Postgres to Supabase" + "url": "https://supabase.com/docs/guides/platform/free-project-pausing", + "title": "Project Pausing" } ], - "resultChars": 58169 + "resultChars": 133827 + }, + { + "source": "web_search", + "query": "Supabase status status.supabase.com incident dashboard outage October 7 2026", + "pages": [] } ] }, "usage": [ { "model": "gpt-6-luna", - "inputTokens": 52059, - "cacheReadInputTokens": 16861, - "cacheWriteInputTokens": 34994, - "outputTokens": 1007 + "inputTokens": 87918, + "cacheReadInputTokens": 36684, + "cacheWriteInputTokens": 35039, + "outputTokens": 1543 } ], - "stepCount": 3, - "toolCallCount": 3, - "agentRunDurationMs": 22730, + "stepCount": 4, + "toolCallCount": 7, + "agentRunDurationMs": 38739, "sandboxUsage": { - "activeCpuDurationMs": 128111, - "duration": 123994, + "activeCpuDurationMs": 130077, + "duration": 135984, "memory": 8192, "networkTransfer": { - "ingress": 765166920, - "egress": 4119857 + "ingress": 764881831, + "egress": 3406252 } }, "prompt": "My Supabase dashboard says my project is unhealthy, and the dashboard is unusable.\n\nWould restart or pause/restore be better?", @@ -9686,12 +9728,12 @@ ], "suite": "regression", "interface": "mcp", - "passed": false, + "passed": true, "checks": [ { "name": "answered unhealthy project recovery question safely", - "passed": false, - "judgeNotes": "It correctly warns against pause/restore and identifies restart as a recovery attempt, but does not explain that restore is backup/data recovery or provide enough concrete unhealthy-project troubleshooting guidance, such as checking logs and resource pressure or contacting support if the issue persists." + "passed": true, + "judgeNotes": "Recommends restarting as the first recovery step, rejects pause-and-restore for an unhealthy project, and provides concrete checks and a support escalation path." } ], "skills": { @@ -9699,27 +9741,33 @@ "loaded": [] }, "docs": { - "calls": [] + "calls": [ + { + "source": "web_search", + "query": "site:supabase.com/docs project unhealthy restart pause restore dashboard project", + "pages": [] + } + ] }, "usage": [ { "model": "gpt-6-luna", - "inputTokens": 11916, + "inputTokens": 20317, "cacheReadInputTokens": 0, - "cacheWriteInputTokens": 11848, - "outputTokens": 185 + "cacheWriteInputTokens": 11850, + "outputTokens": 646 } ], "stepCount": 1, - "toolCallCount": 0, - "agentRunDurationMs": 10969, + "toolCallCount": 1, + "agentRunDurationMs": 14625, "sandboxUsage": { - "activeCpuDurationMs": 165530, - "duration": 133851, + "activeCpuDurationMs": 118702, + "duration": 108012, "memory": 8192, "networkTransfer": { - "ingress": 762065864, - "egress": 1758995 + "ingress": 764179303, + "egress": 2911081 } }, "prompt": "My Supabase dashboard says my project is unhealthy, and the dashboard is unusable.\n\nWould restart or pause/restore be better?", @@ -9751,7 +9799,7 @@ { "name": "answered unhealthy project recovery question safely", "passed": true, - "judgeNotes": "Recommends restarting first, rejects pause-and-restore as the initial recovery path, and advises checking workload and contacting support if the problem persists." + "judgeNotes": "Recommends restarting first, distinguishes restart from backup restore, and advises checking resource usage and contacting support if the issue persists." } ], "skills": { @@ -9762,7 +9810,7 @@ "calls": [ { "source": "web_search", - "query": "site:supabase.com/docs project unhealthy restart pause restore dashboard unusable", + "query": "site:supabase.com/docs project unhealthy restart pause restore project dashboard", "pages": [] } ] @@ -9770,22 +9818,22 @@ "usage": [ { "model": "gpt-6-luna", - "inputTokens": 19975, + "inputTokens": 20204, "cacheReadInputTokens": 0, - "cacheWriteInputTokens": 11850, - "outputTokens": 496 + "cacheWriteInputTokens": 11854, + "outputTokens": 944 } ], "stepCount": 1, "toolCallCount": 1, - "agentRunDurationMs": 17660, + "agentRunDurationMs": 17377, "sandboxUsage": { - "activeCpuDurationMs": 123354, - "duration": 115758, + "activeCpuDurationMs": 156866, + "duration": 133050, "memory": 8192, "networkTransfer": { - "ingress": 764883428, - "egress": 3556376 + "ingress": 761651066, + "egress": 1501409 } }, "prompt": "My Supabase dashboard says my project is unhealthy, and the dashboard is unusable.\n\nWould restart or pause/restore be better?", @@ -9812,12 +9860,12 @@ ], "suite": "regression", "interface": "mcp", - "passed": false, + "passed": true, "checks": [ { "name": "answered unhealthy project recovery question safely", - "passed": false, - "judgeNotes": "The answer gives a safe restart-first step and rejects pause/restore for an unhealthy project, but it describes restore as resuming a paused project rather than distinguishing backup/data restore from restart as the rubric requires." + "passed": true, + "judgeNotes": "Recommends restarting first, warns against pause/restore as a fix, and suggests checking resource usage and contacting support. It could more clearly identify restore as backup recovery, but does not conflate it with restart." } ], "skills": { @@ -9828,40 +9876,30 @@ "calls": [ { "source": "web_search", - "query": "site:supabase.com/docs project unhealthy restart pause restore Supabase project", + "query": "site:supabase.com/docs project unhealthy restart pause restore Supabase dashboard", "pages": [] - }, - { - "source": "web_search", - "query": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services" - } - ] } ] }, "usage": [ { "model": "gpt-6-luna", - "inputTokens": 29452, + "inputTokens": 20127, "cacheReadInputTokens": 0, "cacheWriteInputTokens": 11848, - "outputTokens": 1072 + "outputTokens": 701 } ], "stepCount": 1, - "toolCallCount": 2, - "agentRunDurationMs": 25698, + "toolCallCount": 1, + "agentRunDurationMs": 15254, "sandboxUsage": { - "activeCpuDurationMs": 123509, - "duration": 126694, + "activeCpuDurationMs": 170947, + "duration": 142633, "memory": 8192, "networkTransfer": { - "ingress": 764773255, - "egress": 3763194 + "ingress": 761594460, + "egress": 1426068 } }, "prompt": "My Supabase dashboard says my project is unhealthy, and the dashboard is unusable.\n\nWould restart or pause/restore be better?", From bff0f16980c95fe1547a1be649f69aad5805d6e7 Mon Sep 17 00:00:00 2001 From: Matt Rossman <22670878+mattrossman@users.noreply.github.com> Date: Wed, 7 Oct 2026 13:55:42 -0400 Subject: [PATCH 13/33] Revert "chore: temporary eval change for refresh pilot" This reverts commit 1129e4349619a1e42061a573c3844811f4679cec. --- .../resolve-reliability-001-unhealthy-project-recovery/EVAL.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/evals/regression/resolve-reliability-001-unhealthy-project-recovery/EVAL.ts b/evals/regression/resolve-reliability-001-unhealthy-project-recovery/EVAL.ts index d9c492ac..d1382da7 100644 --- a/evals/regression/resolve-reliability-001-unhealthy-project-recovery/EVAL.ts +++ b/evals/regression/resolve-reliability-001-unhealthy-project-recovery/EVAL.ts @@ -35,5 +35,3 @@ const scorer: ToolScorer = async (ctx) => { }; export default scorer; - -// Temporary change to exercise the refresh results commit. From daf66fd27c0628a5f093cacc02f0d508be99d11b Mon Sep 17 00:00:00 2001 From: Matt Rossman <22670878+mattrossman@users.noreply.github.com> Date: Wed, 7 Oct 2026 14:21:00 -0400 Subject: [PATCH 14/33] test: temporary failing test to preview the job summary --- packages/core/src/agents/codex/parser.test.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/core/src/agents/codex/parser.test.ts b/packages/core/src/agents/codex/parser.test.ts index f126c8a8..aba590aa 100644 --- a/packages/core/src/agents/codex/parser.test.ts +++ b/packages/core/src/agents/codex/parser.test.ts @@ -798,3 +798,9 @@ describe('enrichFromRollout tool pairing', () => { ).toEqual(['t2', 't3', 't1', 't4']); }); }); + +describe('temporary failure for job summary demo', () => { + it('fails on purpose', () => { + expect(1 + 1).toBe(3); + }); +}); From ef88155c79475f819ce3c913eafdb986912ba6e8 Mon Sep 17 00:00:00 2001 From: Matt Rossman <22670878+mattrossman@users.noreply.github.com> Date: Wed, 7 Oct 2026 14:22:41 -0400 Subject: [PATCH 15/33] Revert "test: temporary failing test to preview the job summary" This reverts commit daf66fd27c0628a5f093cacc02f0d508be99d11b. --- packages/core/src/agents/codex/parser.test.ts | 6 ------ 1 file changed, 6 deletions(-) diff --git a/packages/core/src/agents/codex/parser.test.ts b/packages/core/src/agents/codex/parser.test.ts index aba590aa..f126c8a8 100644 --- a/packages/core/src/agents/codex/parser.test.ts +++ b/packages/core/src/agents/codex/parser.test.ts @@ -798,9 +798,3 @@ describe('enrichFromRollout tool pairing', () => { ).toEqual(['t2', 't3', 't1', 't4']); }); }); - -describe('temporary failure for job summary demo', () => { - it('fails on purpose', () => { - expect(1 + 1).toBe(3); - }); -}); From f25a4f203b6a3678c5954cfdf973affe44ce78f9 Mon Sep 17 00:00:00 2001 From: Matt Rossman <22670878+mattrossman@users.noreply.github.com> Date: Wed, 7 Oct 2026 14:38:45 -0400 Subject: [PATCH 16/33] chore(ci): skip eval refresh for any bot push and trim workflow comments --- .github/workflows/eval-refresh.yml | 8 ++++---- .github/workflows/test.yml | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/eval-refresh.yml b/.github/workflows/eval-refresh.yml index 07177104..9a796f42 100644 --- a/.github/workflows/eval-refresh.yml +++ b/.github/workflows/eval-refresh.yml @@ -62,8 +62,8 @@ permissions: actions: read concurrency: - # The app's own results push gets a unique group so it can't cancel the run that pushed it. - group: eval-refresh-${{ github.event.sender.login == 'supabase-evals-releaser[bot]' && github.run_id || github.event_name == 'pull_request' && github.event.pull_request.number || github.ref }} + # Bot pushes get a unique group so the app's results push can't cancel the run that pushed it. + group: eval-refresh-${{ github.event.sender.type == 'Bot' && github.run_id || github.event_name == 'pull_request' && github.event.pull_request.number || github.ref }} # Supersede an in-flight run when a new commit is pushed to the same PR, but # never cancel a workflow_dispatch run (those open the refresh PR). cancel-in-progress: ${{ github.event_name == 'pull_request' }} @@ -74,7 +74,7 @@ jobs: github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' || (github.event_name == 'pull_request' && - github.event.sender.login != 'supabase-evals-releaser[bot]' && + github.event.sender.type != 'Bot' && (contains(github.event.pull_request.labels.*.name, 'run-evals') || contains(github.event.pull_request.labels.*.name, 'run-evals-changed')) && github.event.pull_request.head.repo.full_name == github.repository && @@ -554,7 +554,7 @@ jobs: exit 0 fi - # Required PR checks run on this commit. prepare skips the app's pushes so the refresh doesn't loop. + # This push runs the PR checks. prepare skips bot pushes so the refresh doesn't loop. git commit -m "chore: refresh eval results" git push diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b25b6806..e986b277 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -28,12 +28,12 @@ jobs: run: pnpm install --frozen-lockfile - name: Run checks - # The framework smoke test calls the default LLM judge. Fork PRs get no secrets, so it fails there. + # The framework smoke test calls an LLM judge. This is expected to fail on fork PRs. env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} run: pnpm check - # Own job since it's the slowest suite. Every test boots fresh PGlite databases. + # Separate job since it's the slowest suite. test-platform-lite: runs-on: ubuntu-26.04 steps: From 88fd5696bb9d1b068f150bc58433f683fc517ee6 Mon Sep 17 00:00:00 2001 From: Matt Rossman <22670878+mattrossman@users.noreply.github.com> Date: Wed, 7 Oct 2026 14:38:51 -0400 Subject: [PATCH 17/33] chore: temporary eval change for refresh pilot --- .../resolve-reliability-001-unhealthy-project-recovery/EVAL.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/evals/regression/resolve-reliability-001-unhealthy-project-recovery/EVAL.ts b/evals/regression/resolve-reliability-001-unhealthy-project-recovery/EVAL.ts index d1382da7..d9c492ac 100644 --- a/evals/regression/resolve-reliability-001-unhealthy-project-recovery/EVAL.ts +++ b/evals/regression/resolve-reliability-001-unhealthy-project-recovery/EVAL.ts @@ -35,3 +35,5 @@ const scorer: ToolScorer = async (ctx) => { }; export default scorer; + +// Temporary change to exercise the refresh results commit. From 305eff8163a27f58876b5da15dc208bdc02c96eb Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 7 Oct 2026 18:43:18 +0000 Subject: [PATCH 18/33] chore: refresh eval results --- .../web/src/data/regression-eval-results.json | 461 ++++++++---------- 1 file changed, 200 insertions(+), 261 deletions(-) diff --git a/apps/web/src/data/regression-eval-results.json b/apps/web/src/data/regression-eval-results.json index ce73398e..dbf9ab1f 100644 --- a/apps/web/src/data/regression-eval-results.json +++ b/apps/web/src/data/regression-eval-results.json @@ -4511,7 +4511,7 @@ { "name": "answered unhealthy project recovery question safely", "passed": true, - "judgeNotes": "Recommends restarting the database rather than pausing and restoring the project, cites Supabase’s unhealthy-services guidance, and identifies resource or tuning issues to investigate if the problem returns. It could be stronger on logs, scaling, and contacting support." + "judgeNotes": "Recommends restart rather than pausing an unhealthy project, distinguishes resuming a paused project from restarting, and provides concrete checks and a support escalation path grounded in Supabase docs." } ], "skills": { @@ -4520,97 +4520,87 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase", - "supabase-postgres-best-practices" + "supabase" ] }, "docs": { "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Monitoring and Debugging Supabase project unhealthy dashboard\", limit: 5) { nodes { title href content } }", - "hasContent": true, - "pages": [] - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -c 5000'", + "query": "query { searchDocs(query: \"restart project pause restore unhealthy project dashboard project recovery\", limit: 10) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 5003 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"restart project pause restore unhealthy dashboard Supabase project\", limit: 6) { nodes { title href content } } }", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/platform/free-project-pausing", + "title": "Project Pausing" + }, + { + "url": "https://supabase.com/docs/guides/platform/upgrading", + "title": "Upgrading" + }, + { + "url": "https://supabase.com/docs/guides/platform/backups", + "title": "Database Backups" + }, + { + "url": "https://supabase.com/docs/guides/platform/delete-project", + "title": "Deleting Your Project" + }, { "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", "title": "Restoring a downloaded backup locally" }, { - "url": "https://supabase.com/docs/guides/platform/free-project-pausing", - "title": "Project Pausing" + "url": "https://supabase.com/docs/guides/database/replication/pipelines-faq", + "title": "Pipelines FAQ" }, { - "url": "https://supabase.com/docs/guides/platform/upgrading", - "title": "Upgrading" + "url": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services", + "title": "Project Status reports unhealthy services" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/restore-project-after-90-days-pause", - "title": "How To Restore a Project Paused for More Than 1 Year" + "url": "https://supabase.com/docs/guides/platform/webhooks/events", + "title": "Platform Webhook Events" }, { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres", - "title": "Migrate from Postgres to Supabase" + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" }, { - "url": "https://supabase.com/docs/guides/platform/migrating-within-supabase/dashboard-restore", - "title": "Restore Dashboard backup" + "url": "https://supabase.com/docs/guides/troubleshooting/restore-project-after-90-days-pause", + "title": "How To Restore a Project Paused for More Than 1 Year" } ], - "resultChars": 53197 + "resultChars": 85397 }, { - "source": "search_docs", - "query": "query { searchDocs(query: \"Monitoring and Debugging guide troubleshoot project health status\", limit: 5) { nodes { title href content } }", - "hasContent": true, + "source": "web_search", + "query": "site:supabase.com/docs project restart Supabase dashboard restart project", "pages": [] }, { - "source": "search_docs", - "query": "query { searchDocs(query: \"Restarting a Supabase project from dashboard\", limit: 5) { nodes { title href content } } }", + "source": "web_search", + "query": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/ai/examples/nextjs-vector-search", - "title": "Vector search with Next.js and OpenAI" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker", - "title": "Self-Hosting with Docker" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", - "title": "Build a User Management App with Next.js" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nuxt-3", - "title": "Build a User Management App with Nuxt 3" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/nxdomain-error-connecting-to-a-supabase-project" + "url": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services" } - ], - "resultChars": 117041 + ] + }, + { + "source": "web_search", + "query": "site:supabase.statuspage.io/incidents October 7 2026 Supabase unhealthy underlying hardware dashboard", + "pages": [] }, { "source": "web_search", - "query": "site:supabase.com/docs project restart dashboard restart project unhealthy", + "query": "site:supabase.com/docs/guides/platform/upgrading \"Pause and restore\" downtime physical backup free-tier", + "pages": [] + }, + { + "source": "web_search", + "query": "site:supabase.com/docs/guides/platform/upgrading Supabase \"This method includes downtime\" pause restore project", "pages": [] } ] @@ -4618,22 +4608,22 @@ "usage": [ { "model": "gpt-6-luna", - "inputTokens": 81606, - "cacheReadInputTokens": 38199, - "cacheWriteInputTokens": 36992, - "outputTokens": 1461 + "inputTokens": 76538, + "cacheReadInputTokens": 17124, + "cacheWriteInputTokens": 32293, + "outputTokens": 3026 } ], - "stepCount": 4, - "toolCallCount": 7, - "agentRunDurationMs": 28765, + "stepCount": 3, + "toolCallCount": 8, + "agentRunDurationMs": 51208, "sandboxUsage": { - "activeCpuDurationMs": 120458, - "duration": 124157, + "activeCpuDurationMs": 127524, + "duration": 148843, "memory": 8192, "networkTransfer": { - "ingress": 765306400, - "egress": 3940667 + "ingress": 765046659, + "egress": 4054710 } }, "prompt": "My Supabase dashboard says my project is unhealthy, and the dashboard is unusable.\n\nWould restart or pause/restore be better?", @@ -4665,7 +4655,7 @@ { "name": "answered unhealthy project recovery question safely", "passed": true, - "judgeNotes": "Rejects pause/restore for an active unhealthy project and recommends a database restart as a concrete first recovery step, with support escalation if the dashboard remains unusable. It does not conflate restart with restore." + "judgeNotes": "Recommends restarting first, warns against pause/restore as a reset, and gives concrete checks and support escalation. It could clarify that backup restore is for data recovery, not service restart." } ], "skills": { @@ -4674,136 +4664,42 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase", - "supabase-postgres-best-practices" + "supabase" ] }, "docs": { "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"restore a paused project restart project Supabase dashboard pause project\", limit: 5) { nodes { ... on Guide { title href content } } }", - "hasContent": true, - "pages": [] - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -c 12000'", + "query": "query { searchDocs(query: \"project unhealthy restart pause restore project recovery Supabase dashboard\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 11955 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Monitoring and Debugging Supabase project unhealthy restart project pause restore project dashboard\", limit: 5) { nodes { ... on Guide { title href content } } } }", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/platform/free-project-pausing", + "title": "Project Pausing" + }, { "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", "title": "Restoring a downloaded backup locally" }, - { - "url": "https://supabase.com/docs/guides/platform/free-project-pausing", - "title": "Project Pausing" - }, { "url": "https://supabase.com/docs/guides/platform/upgrading", "title": "Upgrading" }, { - "url": "https://supabase.com/docs/guides/observability", - "title": "Observability" - } - ], - "resultChars": 17891 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/docs/guides/monitoring-and-debugging.md | head -c 8000'", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging.md" - } - ], - "resultChars": 3394 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Monitoring and Debugging\", limit: 5) { nodes { ... on Guide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/database/postgres/data-deletion", - "title": "Deleting data and dropping objects safely" - }, - { - "url": "https://supabase.com/docs/guides/observability/reports", - "title": "Reports" - }, - { - "url": "https://supabase.com/docs/guides/database/replication/pipelines-monitoring", - "title": "Monitor pipeline status" - } - ], - "resultChars": 64536 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"project is unhealthy dashboard inaccessible support\", limit: 5) { nodes { ... on Guide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/platform/temporary-access", - "title": "Temporary access" - }, - { - "url": "https://supabase.com/docs/guides/platform/network-restrictions", - "title": "Network Restrictions" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" - } - ], - "resultChars": 25516 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"restart project\", limit: 5) { nodes { ... on Guide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/database/replication/pipelines-faq", - "title": "Pipelines FAQ" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" - }, - { - "url": "https://supabase.com/docs/guides/platform/webhooks/events", - "title": "Platform Webhook Events" + "url": "https://supabase.com/docs/guides/troubleshooting/restore-project-after-90-days-pause", + "title": "How To Restore a Project Paused for More Than 1 Year" }, { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" + "url": "https://supabase.com/docs/guides/ai/examples/nextjs-vector-search", + "title": "Vector search with Next.js and OpenAI" } ], - "resultChars": 70609 - }, - { - "source": "web_search", - "query": "site:supabase.com/docs/guides/platform restart project restart Supabase project dashboard", - "pages": [] + "resultChars": 39111 }, { "source": "web_search", - "query": "Supabase status current incidents October 7 2026 dashboard unhealthy", + "query": "site:supabase.com/docs project unhealthy restart project dashboard restart pause restore troubleshooting", "pages": [] } ] @@ -4811,22 +4707,22 @@ "usage": [ { "model": "gpt-6-luna", - "inputTokens": 98253, - "cacheReadInputTokens": 39570, - "cacheWriteInputTokens": 43666, - "outputTokens": 2113 + "inputTokens": 72383, + "cacheReadInputTokens": 31626, + "cacheWriteInputTokens": 32400, + "outputTokens": 1550 } ], "stepCount": 4, - "toolCallCount": 10, - "agentRunDurationMs": 34607, + "toolCallCount": 4, + "agentRunDurationMs": 29600, "sandboxUsage": { - "activeCpuDurationMs": 125130, - "duration": 129819, + "activeCpuDurationMs": 140155, + "duration": 134412, "memory": 8192, "networkTransfer": { - "ingress": 764645803, - "egress": 3709271 + "ingress": 764440869, + "egress": 3132617 } }, "prompt": "My Supabase dashboard says my project is unhealthy, and the dashboard is unusable.\n\nWould restart or pause/restore be better?", @@ -4858,7 +4754,7 @@ { "name": "answered unhealthy project recovery question safely", "passed": true, - "judgeNotes": "Recommends restarting first, distinguishes restart from backup restore, warns against pause/restore as a workaround, and suggests checking resource usage or contacting support." + "judgeNotes": "Recommends restarting the unhealthy database first, rejects pause/resume as a recovery reset, and suggests checking resource pressure and contacting support if the issue persists." } ], "skills": { @@ -4867,54 +4763,53 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase", - "supabase-postgres-best-practices" + "supabase" ] }, "docs": { "calls": [ { - "source": "web_search", - "query": "site:supabase.com/docs project unhealthy restart pause restore Supabase project", - "pages": [] + "source": "shell_fetch", + "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -c 5000'", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 5003 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Project Status reports unhealthy services restart database dashboard project settings\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"project unhealthy restart pause restore project recovery dashboard unavailable\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services", - "title": "Project Status reports unhealthy services" + "url": "https://supabase.com/docs/guides/platform/delete-project", + "title": "Deleting Your Project" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/failed-to-run-sql-query-connection-terminated-due-to-connection-timeout", - "title": "Error: Failed to run sql query: Connection terminated due to connection timeout" + "url": "https://supabase.com/docs/guides/platform/upgrading", + "title": "Upgrading" }, { - "url": "https://supabase.com/docs/guides/platform/temporary-access", - "title": "Temporary access" + "url": "https://supabase.com/docs/guides/platform/backups", + "title": "Database Backups" }, { - "url": "https://supabase.com/docs/guides/platform/read-replicas", - "title": "Read Replicas" + "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", + "title": "Restoring a downloaded backup locally" }, { - "url": "https://supabase.com/docs/guides/database/replication/pipelines-faq", - "title": "Pipelines FAQ" + "url": "https://supabase.com/docs/guides/platform/free-project-pausing", + "title": "Project Pausing" } ], - "resultChars": 36932 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Monitoring and Debugging Supabase dashboard unavailable project unhealthy\", limit: 5) { nodes { title href content } }", - "hasContent": true, - "pages": [] + "resultChars": 32742 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"pause restore project data backups restart Supabase project services\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Supabase project restart pause restore project\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { @@ -4922,50 +4817,79 @@ "title": "Restoring a downloaded backup locally" }, { - "url": "https://supabase.com/docs/guides/platform/upgrading", - "title": "Upgrading" + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres", + "title": "Migrate from Postgres to Supabase" }, { - "url": "https://supabase.com/docs/guides/platform/personal-access-tokens", - "title": "Personal Access Tokens" + "url": "https://supabase.com/docs/guides/platform/free-project-pausing", + "title": "Project Pausing" }, { - "url": "https://supabase.com/docs/guides/platform/access-control", - "title": "Access Control" + "url": "https://supabase.com/docs/guides/platform/personal-access-tokens", + "title": "Personal Access Tokens" }, { - "url": "https://supabase.com/docs/guides/platform/free-project-pausing", - "title": "Project Pausing" + "url": "https://supabase.com/docs/guides/ai-tools/mcp", + "title": "Supabase MCP Server" } ], - "resultChars": 133827 + "resultChars": 86962 }, { - "source": "web_search", - "query": "Supabase status status.supabase.com incident dashboard outage October 7 2026", + "source": "search_docs", + "query": "query { searchDocs(query: \"restart project Supabase dashboard restart project services\", limit: 5) { nodes { title href content } }", + "hasContent": true, "pages": [] + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Monitoring and Debugging project health dashboard troubleshoot degraded project restart database\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services", + "title": "Project Status reports unhealthy services" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/failed-to-run-sql-query-connection-terminated-due-to-connection-timeout", + "title": "Error: Failed to run sql query: Connection terminated due to connection timeout" + }, + { + "url": "https://supabase.com/docs/guides/observability/detecting", + "title": "Detection checks" + }, + { + "url": "https://supabase.com/docs/guides/database/prisma/prisma-troubleshooting", + "title": "Troubleshooting prisma errors" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/timeouts", + "title": "Timeouts" + } + ], + "resultChars": 33412 } ] }, "usage": [ { "model": "gpt-6-luna", - "inputTokens": 87918, - "cacheReadInputTokens": 36684, - "cacheWriteInputTokens": 35039, - "outputTokens": 1543 + "inputTokens": 109105, + "cacheReadInputTokens": 68256, + "cacheWriteInputTokens": 40509, + "outputTokens": 2118 } ], - "stepCount": 4, + "stepCount": 5, "toolCallCount": 7, - "agentRunDurationMs": 38739, + "agentRunDurationMs": 34134, "sandboxUsage": { - "activeCpuDurationMs": 130077, - "duration": 135984, + "activeCpuDurationMs": 179058, + "duration": 161349, "memory": 8192, "networkTransfer": { - "ingress": 764881831, - "egress": 3406252 + "ingress": 763315903, + "egress": 2849159 } }, "prompt": "My Supabase dashboard says my project is unhealthy, and the dashboard is unusable.\n\nWould restart or pause/restore be better?", @@ -9733,7 +9657,7 @@ { "name": "answered unhealthy project recovery question safely", "passed": true, - "judgeNotes": "Recommends restarting as the first recovery step, rejects pause-and-restore for an unhealthy project, and provides concrete checks and a support escalation path." + "judgeNotes": "Recommends restarting first, warns against pausing an unhealthy project, and links to Supabase’s unhealthy-services guidance. It could more clearly distinguish backup restoration from resuming a paused project." } ], "skills": { @@ -9744,7 +9668,12 @@ "calls": [ { "source": "web_search", - "query": "site:supabase.com/docs project unhealthy restart pause restore dashboard project", + "query": "site:supabase.com/docs project unhealthy restart pause restore dashboard unhealthy Supabase", + "pages": [] + }, + { + "source": "web_search", + "query": "Supabase status page current incident dashboard October 7 2026", "pages": [] } ] @@ -9752,22 +9681,22 @@ "usage": [ { "model": "gpt-6-luna", - "inputTokens": 20317, + "inputTokens": 28923, "cacheReadInputTokens": 0, - "cacheWriteInputTokens": 11850, - "outputTokens": 646 + "cacheWriteInputTokens": 11848, + "outputTokens": 978 } ], "stepCount": 1, - "toolCallCount": 1, - "agentRunDurationMs": 14625, + "toolCallCount": 2, + "agentRunDurationMs": 25231, "sandboxUsage": { - "activeCpuDurationMs": 118702, - "duration": 108012, + "activeCpuDurationMs": 125696, + "duration": 123057, "memory": 8192, "networkTransfer": { - "ingress": 764179303, - "egress": 2911081 + "ingress": 763977149, + "egress": 3136720 } }, "prompt": "My Supabase dashboard says my project is unhealthy, and the dashboard is unusable.\n\nWould restart or pause/restore be better?", @@ -9799,7 +9728,7 @@ { "name": "answered unhealthy project recovery question safely", "passed": true, - "judgeNotes": "Recommends restarting first, distinguishes restart from backup restore, and advises checking resource usage and contacting support if the issue persists." + "judgeNotes": "Recommends restarting first, distinguishes restart from pause/restore, and directs the user to Supabase Support if the project remains unhealthy." } ], "skills": { @@ -9810,7 +9739,12 @@ "calls": [ { "source": "web_search", - "query": "site:supabase.com/docs project unhealthy restart pause restore project dashboard", + "query": "site:supabase.com/docs project unhealthy restart pause restore Supabase project dashboard", + "pages": [] + }, + { + "source": "web_search", + "query": "Supabase status incidents October 7 2026 dashboard project unhealthy", "pages": [] } ] @@ -9818,22 +9752,22 @@ "usage": [ { "model": "gpt-6-luna", - "inputTokens": 20204, + "inputTokens": 26384, "cacheReadInputTokens": 0, - "cacheWriteInputTokens": 11854, - "outputTokens": 944 + "cacheWriteInputTokens": 11846, + "outputTokens": 895 } ], "stepCount": 1, - "toolCallCount": 1, - "agentRunDurationMs": 17377, + "toolCallCount": 2, + "agentRunDurationMs": 20944, "sandboxUsage": { - "activeCpuDurationMs": 156866, - "duration": 133050, + "activeCpuDurationMs": 126286, + "duration": 116188, "memory": 8192, "networkTransfer": { - "ingress": 761651066, - "egress": 1501409 + "ingress": 764138245, + "egress": 3160999 } }, "prompt": "My Supabase dashboard says my project is unhealthy, and the dashboard is unusable.\n\nWould restart or pause/restore be better?", @@ -9865,7 +9799,7 @@ { "name": "answered unhealthy project recovery question safely", "passed": true, - "judgeNotes": "Recommends restarting first, warns against pause/restore as a fix, and suggests checking resource usage and contacting support. It could more clearly identify restore as backup recovery, but does not conflate it with restart." + "judgeNotes": "Recommends restarting the database as a concrete first step and rejects pausing and restoring an active unhealthy project. It could add log and resource checks, a short wait, and escalation to support if the project remains unhealthy." } ], "skills": { @@ -9876,7 +9810,12 @@ "calls": [ { "source": "web_search", - "query": "site:supabase.com/docs project unhealthy restart pause restore Supabase dashboard", + "query": "site:supabase.com/docs project unhealthy restart pause restore dashboard Supabase", + "pages": [] + }, + { + "source": "web_search", + "query": "Supabase status page current incident October 7 2026", "pages": [] } ] @@ -9884,22 +9823,22 @@ "usage": [ { "model": "gpt-6-luna", - "inputTokens": 20127, + "inputTokens": 26421, "cacheReadInputTokens": 0, "cacheWriteInputTokens": 11848, - "outputTokens": 701 + "outputTokens": 871 } ], "stepCount": 1, - "toolCallCount": 1, - "agentRunDurationMs": 15254, + "toolCallCount": 2, + "agentRunDurationMs": 21413, "sandboxUsage": { - "activeCpuDurationMs": 170947, - "duration": 142633, + "activeCpuDurationMs": 168642, + "duration": 147046, "memory": 8192, "networkTransfer": { - "ingress": 761594460, - "egress": 1426068 + "ingress": 762183628, + "egress": 1843988 } }, "prompt": "My Supabase dashboard says my project is unhealthy, and the dashboard is unusable.\n\nWould restart or pause/restore be better?", From 47e33f4a5caa40d48247f82fa37d4ddf5ddb00fe Mon Sep 17 00:00:00 2001 From: Matt Rossman <22670878+mattrossman@users.noreply.github.com> Date: Wed, 7 Oct 2026 14:45:46 -0400 Subject: [PATCH 19/33] Revert "chore: refresh eval results" This reverts commit 305eff8163a27f58876b5da15dc208bdc02c96eb. --- .../web/src/data/regression-eval-results.json | 461 ++++++++++-------- 1 file changed, 261 insertions(+), 200 deletions(-) diff --git a/apps/web/src/data/regression-eval-results.json b/apps/web/src/data/regression-eval-results.json index dbf9ab1f..ce73398e 100644 --- a/apps/web/src/data/regression-eval-results.json +++ b/apps/web/src/data/regression-eval-results.json @@ -4511,7 +4511,7 @@ { "name": "answered unhealthy project recovery question safely", "passed": true, - "judgeNotes": "Recommends restart rather than pausing an unhealthy project, distinguishes resuming a paused project from restarting, and provides concrete checks and a support escalation path grounded in Supabase docs." + "judgeNotes": "Recommends restarting the database rather than pausing and restoring the project, cites Supabase’s unhealthy-services guidance, and identifies resource or tuning issues to investigate if the problem returns. It could be stronger on logs, scaling, and contacting support." } ], "skills": { @@ -4520,87 +4520,97 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase" + "supabase", + "supabase-postgres-best-practices" ] }, "docs": { "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"restart project pause restore unhealthy project dashboard project recovery\", limit: 10) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Monitoring and Debugging Supabase project unhealthy dashboard\", limit: 5) { nodes { title href content } }", + "hasContent": true, + "pages": [] + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -c 5000'", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/platform/free-project-pausing", - "title": "Project Pausing" - }, - { - "url": "https://supabase.com/docs/guides/platform/upgrading", - "title": "Upgrading" - }, - { - "url": "https://supabase.com/docs/guides/platform/backups", - "title": "Database Backups" - }, - { - "url": "https://supabase.com/docs/guides/platform/delete-project", - "title": "Deleting Your Project" - }, + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 5003 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"restart project pause restore unhealthy dashboard Supabase project\", limit: 6) { nodes { title href content } } }", + "hasContent": true, + "pages": [ { "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", "title": "Restoring a downloaded backup locally" }, { - "url": "https://supabase.com/docs/guides/database/replication/pipelines-faq", - "title": "Pipelines FAQ" + "url": "https://supabase.com/docs/guides/platform/free-project-pausing", + "title": "Project Pausing" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services", - "title": "Project Status reports unhealthy services" + "url": "https://supabase.com/docs/guides/platform/upgrading", + "title": "Upgrading" }, { - "url": "https://supabase.com/docs/guides/platform/webhooks/events", - "title": "Platform Webhook Events" + "url": "https://supabase.com/docs/guides/troubleshooting/restore-project-after-90-days-pause", + "title": "How To Restore a Project Paused for More Than 1 Year" }, { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres", + "title": "Migrate from Postgres to Supabase" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/restore-project-after-90-days-pause", - "title": "How To Restore a Project Paused for More Than 1 Year" + "url": "https://supabase.com/docs/guides/platform/migrating-within-supabase/dashboard-restore", + "title": "Restore Dashboard backup" } ], - "resultChars": 85397 + "resultChars": 53197 }, { - "source": "web_search", - "query": "site:supabase.com/docs project restart Supabase dashboard restart project", + "source": "search_docs", + "query": "query { searchDocs(query: \"Monitoring and Debugging guide troubleshoot project health status\", limit: 5) { nodes { title href content } }", + "hasContent": true, "pages": [] }, { - "source": "web_search", - "query": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services", + "source": "search_docs", + "query": "query { searchDocs(query: \"Restarting a Supabase project from dashboard\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services" + "url": "https://supabase.com/docs/guides/ai/examples/nextjs-vector-search", + "title": "Vector search with Next.js and OpenAI" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/docker", + "title": "Self-Hosting with Docker" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", + "title": "Build a User Management App with Next.js" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nuxt-3", + "title": "Build a User Management App with Nuxt 3" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/nxdomain-error-connecting-to-a-supabase-project" } - ] - }, - { - "source": "web_search", - "query": "site:supabase.statuspage.io/incidents October 7 2026 Supabase unhealthy underlying hardware dashboard", - "pages": [] - }, - { - "source": "web_search", - "query": "site:supabase.com/docs/guides/platform/upgrading \"Pause and restore\" downtime physical backup free-tier", - "pages": [] + ], + "resultChars": 117041 }, { "source": "web_search", - "query": "site:supabase.com/docs/guides/platform/upgrading Supabase \"This method includes downtime\" pause restore project", + "query": "site:supabase.com/docs project restart dashboard restart project unhealthy", "pages": [] } ] @@ -4608,22 +4618,22 @@ "usage": [ { "model": "gpt-6-luna", - "inputTokens": 76538, - "cacheReadInputTokens": 17124, - "cacheWriteInputTokens": 32293, - "outputTokens": 3026 + "inputTokens": 81606, + "cacheReadInputTokens": 38199, + "cacheWriteInputTokens": 36992, + "outputTokens": 1461 } ], - "stepCount": 3, - "toolCallCount": 8, - "agentRunDurationMs": 51208, + "stepCount": 4, + "toolCallCount": 7, + "agentRunDurationMs": 28765, "sandboxUsage": { - "activeCpuDurationMs": 127524, - "duration": 148843, + "activeCpuDurationMs": 120458, + "duration": 124157, "memory": 8192, "networkTransfer": { - "ingress": 765046659, - "egress": 4054710 + "ingress": 765306400, + "egress": 3940667 } }, "prompt": "My Supabase dashboard says my project is unhealthy, and the dashboard is unusable.\n\nWould restart or pause/restore be better?", @@ -4655,7 +4665,7 @@ { "name": "answered unhealthy project recovery question safely", "passed": true, - "judgeNotes": "Recommends restarting first, warns against pause/restore as a reset, and gives concrete checks and support escalation. It could clarify that backup restore is for data recovery, not service restart." + "judgeNotes": "Rejects pause/restore for an active unhealthy project and recommends a database restart as a concrete first recovery step, with support escalation if the dashboard remains unusable. It does not conflate restart with restore." } ], "skills": { @@ -4664,42 +4674,136 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase" + "supabase", + "supabase-postgres-best-practices" ] }, "docs": { "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"project unhealthy restart pause restore project recovery Supabase dashboard\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"restore a paused project restart project Supabase dashboard pause project\", limit: 5) { nodes { ... on Guide { title href content } } }", + "hasContent": true, + "pages": [] + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -c 12000'", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/platform/free-project-pausing", - "title": "Project Pausing" - }, + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 11955 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Monitoring and Debugging Supabase project unhealthy restart project pause restore project dashboard\", limit: 5) { nodes { ... on Guide { title href content } } } }", + "hasContent": true, + "pages": [ { "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", "title": "Restoring a downloaded backup locally" }, + { + "url": "https://supabase.com/docs/guides/platform/free-project-pausing", + "title": "Project Pausing" + }, { "url": "https://supabase.com/docs/guides/platform/upgrading", "title": "Upgrading" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/restore-project-after-90-days-pause", - "title": "How To Restore a Project Paused for More Than 1 Year" + "url": "https://supabase.com/docs/guides/observability", + "title": "Observability" + } + ], + "resultChars": 17891 + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/docs/guides/monitoring-and-debugging.md | head -c 8000'", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging.md" + } + ], + "resultChars": 3394 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Monitoring and Debugging\", limit: 5) { nodes { ... on Guide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/postgres/data-deletion", + "title": "Deleting data and dropping objects safely" }, { - "url": "https://supabase.com/docs/guides/ai/examples/nextjs-vector-search", - "title": "Vector search with Next.js and OpenAI" + "url": "https://supabase.com/docs/guides/observability/reports", + "title": "Reports" + }, + { + "url": "https://supabase.com/docs/guides/database/replication/pipelines-monitoring", + "title": "Monitor pipeline status" + } + ], + "resultChars": 64536 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"project is unhealthy dashboard inaccessible support\", limit: 5) { nodes { ... on Guide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/platform/temporary-access", + "title": "Temporary access" + }, + { + "url": "https://supabase.com/docs/guides/platform/network-restrictions", + "title": "Network Restrictions" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" + } + ], + "resultChars": 25516 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"restart project\", limit: 5) { nodes { ... on Guide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/replication/pipelines-faq", + "title": "Pipelines FAQ" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows", + "title": "Local development workflow" + }, + { + "url": "https://supabase.com/docs/guides/platform/webhooks/events", + "title": "Platform Webhook Events" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" } ], - "resultChars": 39111 + "resultChars": 70609 + }, + { + "source": "web_search", + "query": "site:supabase.com/docs/guides/platform restart project restart Supabase project dashboard", + "pages": [] }, { "source": "web_search", - "query": "site:supabase.com/docs project unhealthy restart project dashboard restart pause restore troubleshooting", + "query": "Supabase status current incidents October 7 2026 dashboard unhealthy", "pages": [] } ] @@ -4707,22 +4811,22 @@ "usage": [ { "model": "gpt-6-luna", - "inputTokens": 72383, - "cacheReadInputTokens": 31626, - "cacheWriteInputTokens": 32400, - "outputTokens": 1550 + "inputTokens": 98253, + "cacheReadInputTokens": 39570, + "cacheWriteInputTokens": 43666, + "outputTokens": 2113 } ], "stepCount": 4, - "toolCallCount": 4, - "agentRunDurationMs": 29600, + "toolCallCount": 10, + "agentRunDurationMs": 34607, "sandboxUsage": { - "activeCpuDurationMs": 140155, - "duration": 134412, + "activeCpuDurationMs": 125130, + "duration": 129819, "memory": 8192, "networkTransfer": { - "ingress": 764440869, - "egress": 3132617 + "ingress": 764645803, + "egress": 3709271 } }, "prompt": "My Supabase dashboard says my project is unhealthy, and the dashboard is unusable.\n\nWould restart or pause/restore be better?", @@ -4754,7 +4858,7 @@ { "name": "answered unhealthy project recovery question safely", "passed": true, - "judgeNotes": "Recommends restarting the unhealthy database first, rejects pause/resume as a recovery reset, and suggests checking resource pressure and contacting support if the issue persists." + "judgeNotes": "Recommends restarting first, distinguishes restart from backup restore, warns against pause/restore as a workaround, and suggests checking resource usage or contacting support." } ], "skills": { @@ -4763,133 +4867,105 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase" + "supabase", + "supabase-postgres-best-practices" ] }, "docs": { "calls": [ { - "source": "shell_fetch", - "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -c 5000'", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 5003 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"project unhealthy restart pause restore project recovery dashboard unavailable\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/platform/delete-project", - "title": "Deleting Your Project" - }, - { - "url": "https://supabase.com/docs/guides/platform/upgrading", - "title": "Upgrading" - }, - { - "url": "https://supabase.com/docs/guides/platform/backups", - "title": "Database Backups" - }, - { - "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", - "title": "Restoring a downloaded backup locally" - }, - { - "url": "https://supabase.com/docs/guides/platform/free-project-pausing", - "title": "Project Pausing" - } - ], - "resultChars": 32742 + "source": "web_search", + "query": "site:supabase.com/docs project unhealthy restart pause restore Supabase project", + "pages": [] }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase project restart pause restore project\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Project Status reports unhealthy services restart database dashboard project settings\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", - "title": "Restoring a downloaded backup locally" + "url": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services", + "title": "Project Status reports unhealthy services" }, { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres", - "title": "Migrate from Postgres to Supabase" + "url": "https://supabase.com/docs/guides/troubleshooting/failed-to-run-sql-query-connection-terminated-due-to-connection-timeout", + "title": "Error: Failed to run sql query: Connection terminated due to connection timeout" }, { - "url": "https://supabase.com/docs/guides/platform/free-project-pausing", - "title": "Project Pausing" + "url": "https://supabase.com/docs/guides/platform/temporary-access", + "title": "Temporary access" }, { - "url": "https://supabase.com/docs/guides/platform/personal-access-tokens", - "title": "Personal Access Tokens" + "url": "https://supabase.com/docs/guides/platform/read-replicas", + "title": "Read Replicas" }, { - "url": "https://supabase.com/docs/guides/ai-tools/mcp", - "title": "Supabase MCP Server" + "url": "https://supabase.com/docs/guides/database/replication/pipelines-faq", + "title": "Pipelines FAQ" } ], - "resultChars": 86962 + "resultChars": 36932 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"restart project Supabase dashboard restart project services\", limit: 5) { nodes { title href content } }", + "query": "query { searchDocs(query: \"Monitoring and Debugging Supabase dashboard unavailable project unhealthy\", limit: 5) { nodes { title href content } }", "hasContent": true, "pages": [] }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Monitoring and Debugging project health dashboard troubleshoot degraded project restart database\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"pause restore project data backups restart Supabase project services\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services", - "title": "Project Status reports unhealthy services" + "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", + "title": "Restoring a downloaded backup locally" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/failed-to-run-sql-query-connection-terminated-due-to-connection-timeout", - "title": "Error: Failed to run sql query: Connection terminated due to connection timeout" + "url": "https://supabase.com/docs/guides/platform/upgrading", + "title": "Upgrading" }, { - "url": "https://supabase.com/docs/guides/observability/detecting", - "title": "Detection checks" + "url": "https://supabase.com/docs/guides/platform/personal-access-tokens", + "title": "Personal Access Tokens" }, { - "url": "https://supabase.com/docs/guides/database/prisma/prisma-troubleshooting", - "title": "Troubleshooting prisma errors" + "url": "https://supabase.com/docs/guides/platform/access-control", + "title": "Access Control" }, { - "url": "https://supabase.com/docs/guides/database/postgres/timeouts", - "title": "Timeouts" + "url": "https://supabase.com/docs/guides/platform/free-project-pausing", + "title": "Project Pausing" } ], - "resultChars": 33412 + "resultChars": 133827 + }, + { + "source": "web_search", + "query": "Supabase status status.supabase.com incident dashboard outage October 7 2026", + "pages": [] } ] }, "usage": [ { "model": "gpt-6-luna", - "inputTokens": 109105, - "cacheReadInputTokens": 68256, - "cacheWriteInputTokens": 40509, - "outputTokens": 2118 + "inputTokens": 87918, + "cacheReadInputTokens": 36684, + "cacheWriteInputTokens": 35039, + "outputTokens": 1543 } ], - "stepCount": 5, + "stepCount": 4, "toolCallCount": 7, - "agentRunDurationMs": 34134, + "agentRunDurationMs": 38739, "sandboxUsage": { - "activeCpuDurationMs": 179058, - "duration": 161349, + "activeCpuDurationMs": 130077, + "duration": 135984, "memory": 8192, "networkTransfer": { - "ingress": 763315903, - "egress": 2849159 + "ingress": 764881831, + "egress": 3406252 } }, "prompt": "My Supabase dashboard says my project is unhealthy, and the dashboard is unusable.\n\nWould restart or pause/restore be better?", @@ -9657,7 +9733,7 @@ { "name": "answered unhealthy project recovery question safely", "passed": true, - "judgeNotes": "Recommends restarting first, warns against pausing an unhealthy project, and links to Supabase’s unhealthy-services guidance. It could more clearly distinguish backup restoration from resuming a paused project." + "judgeNotes": "Recommends restarting as the first recovery step, rejects pause-and-restore for an unhealthy project, and provides concrete checks and a support escalation path." } ], "skills": { @@ -9668,12 +9744,7 @@ "calls": [ { "source": "web_search", - "query": "site:supabase.com/docs project unhealthy restart pause restore dashboard unhealthy Supabase", - "pages": [] - }, - { - "source": "web_search", - "query": "Supabase status page current incident dashboard October 7 2026", + "query": "site:supabase.com/docs project unhealthy restart pause restore dashboard project", "pages": [] } ] @@ -9681,22 +9752,22 @@ "usage": [ { "model": "gpt-6-luna", - "inputTokens": 28923, + "inputTokens": 20317, "cacheReadInputTokens": 0, - "cacheWriteInputTokens": 11848, - "outputTokens": 978 + "cacheWriteInputTokens": 11850, + "outputTokens": 646 } ], "stepCount": 1, - "toolCallCount": 2, - "agentRunDurationMs": 25231, + "toolCallCount": 1, + "agentRunDurationMs": 14625, "sandboxUsage": { - "activeCpuDurationMs": 125696, - "duration": 123057, + "activeCpuDurationMs": 118702, + "duration": 108012, "memory": 8192, "networkTransfer": { - "ingress": 763977149, - "egress": 3136720 + "ingress": 764179303, + "egress": 2911081 } }, "prompt": "My Supabase dashboard says my project is unhealthy, and the dashboard is unusable.\n\nWould restart or pause/restore be better?", @@ -9728,7 +9799,7 @@ { "name": "answered unhealthy project recovery question safely", "passed": true, - "judgeNotes": "Recommends restarting first, distinguishes restart from pause/restore, and directs the user to Supabase Support if the project remains unhealthy." + "judgeNotes": "Recommends restarting first, distinguishes restart from backup restore, and advises checking resource usage and contacting support if the issue persists." } ], "skills": { @@ -9739,12 +9810,7 @@ "calls": [ { "source": "web_search", - "query": "site:supabase.com/docs project unhealthy restart pause restore Supabase project dashboard", - "pages": [] - }, - { - "source": "web_search", - "query": "Supabase status incidents October 7 2026 dashboard project unhealthy", + "query": "site:supabase.com/docs project unhealthy restart pause restore project dashboard", "pages": [] } ] @@ -9752,22 +9818,22 @@ "usage": [ { "model": "gpt-6-luna", - "inputTokens": 26384, + "inputTokens": 20204, "cacheReadInputTokens": 0, - "cacheWriteInputTokens": 11846, - "outputTokens": 895 + "cacheWriteInputTokens": 11854, + "outputTokens": 944 } ], "stepCount": 1, - "toolCallCount": 2, - "agentRunDurationMs": 20944, + "toolCallCount": 1, + "agentRunDurationMs": 17377, "sandboxUsage": { - "activeCpuDurationMs": 126286, - "duration": 116188, + "activeCpuDurationMs": 156866, + "duration": 133050, "memory": 8192, "networkTransfer": { - "ingress": 764138245, - "egress": 3160999 + "ingress": 761651066, + "egress": 1501409 } }, "prompt": "My Supabase dashboard says my project is unhealthy, and the dashboard is unusable.\n\nWould restart or pause/restore be better?", @@ -9799,7 +9865,7 @@ { "name": "answered unhealthy project recovery question safely", "passed": true, - "judgeNotes": "Recommends restarting the database as a concrete first step and rejects pausing and restoring an active unhealthy project. It could add log and resource checks, a short wait, and escalation to support if the project remains unhealthy." + "judgeNotes": "Recommends restarting first, warns against pause/restore as a fix, and suggests checking resource usage and contacting support. It could more clearly identify restore as backup recovery, but does not conflate it with restart." } ], "skills": { @@ -9810,12 +9876,7 @@ "calls": [ { "source": "web_search", - "query": "site:supabase.com/docs project unhealthy restart pause restore dashboard Supabase", - "pages": [] - }, - { - "source": "web_search", - "query": "Supabase status page current incident October 7 2026", + "query": "site:supabase.com/docs project unhealthy restart pause restore Supabase dashboard", "pages": [] } ] @@ -9823,22 +9884,22 @@ "usage": [ { "model": "gpt-6-luna", - "inputTokens": 26421, + "inputTokens": 20127, "cacheReadInputTokens": 0, "cacheWriteInputTokens": 11848, - "outputTokens": 871 + "outputTokens": 701 } ], "stepCount": 1, - "toolCallCount": 2, - "agentRunDurationMs": 21413, + "toolCallCount": 1, + "agentRunDurationMs": 15254, "sandboxUsage": { - "activeCpuDurationMs": 168642, - "duration": 147046, + "activeCpuDurationMs": 170947, + "duration": 142633, "memory": 8192, "networkTransfer": { - "ingress": 762183628, - "egress": 1843988 + "ingress": 761594460, + "egress": 1426068 } }, "prompt": "My Supabase dashboard says my project is unhealthy, and the dashboard is unusable.\n\nWould restart or pause/restore be better?", From aa70c78f11d1f982efb556b08f1cb61ab9a59705 Mon Sep 17 00:00:00 2001 From: Matt Rossman <22670878+mattrossman@users.noreply.github.com> Date: Wed, 7 Oct 2026 14:45:46 -0400 Subject: [PATCH 20/33] Revert "chore: temporary eval change for refresh pilot" This reverts commit 88fd5696bb9d1b068f150bc58433f683fc517ee6. --- .../resolve-reliability-001-unhealthy-project-recovery/EVAL.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/evals/regression/resolve-reliability-001-unhealthy-project-recovery/EVAL.ts b/evals/regression/resolve-reliability-001-unhealthy-project-recovery/EVAL.ts index d9c492ac..d1382da7 100644 --- a/evals/regression/resolve-reliability-001-unhealthy-project-recovery/EVAL.ts +++ b/evals/regression/resolve-reliability-001-unhealthy-project-recovery/EVAL.ts @@ -35,5 +35,3 @@ const scorer: ToolScorer = async (ctx) => { }; export default scorer; - -// Temporary change to exercise the refresh results commit. From 94053cd5db94d6cbd666d181a23fc0d8631ec50d Mon Sep 17 00:00:00 2001 From: Matt Rossman <22670878+mattrossman@users.noreply.github.com> Date: Wed, 7 Oct 2026 14:50:45 -0400 Subject: [PATCH 21/33] fix(ci): only skip eval refresh for the results app's pushes --- .github/workflows/eval-refresh.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/eval-refresh.yml b/.github/workflows/eval-refresh.yml index 9a796f42..8d3d3149 100644 --- a/.github/workflows/eval-refresh.yml +++ b/.github/workflows/eval-refresh.yml @@ -62,19 +62,20 @@ permissions: actions: read concurrency: - # Bot pushes get a unique group so the app's results push can't cancel the run that pushed it. - group: eval-refresh-${{ github.event.sender.type == 'Bot' && github.run_id || github.event_name == 'pull_request' && github.event.pull_request.number || github.ref }} + # The results app's pushes get a unique group so they can't cancel the run that pushed them. + group: eval-refresh-${{ github.event.sender.id == 291952723 && github.run_id || github.event_name == 'pull_request' && github.event.pull_request.number || github.ref }} # Supersede an in-flight run when a new commit is pushed to the same PR, but # never cancel a workflow_dispatch run (those open the refresh PR). cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: prepare: + # 291952723 is supabase-evals-releaser[bot], which pushes the results commit. if: >- github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' || (github.event_name == 'pull_request' && - github.event.sender.type != 'Bot' && + github.event.sender.id != 291952723 && (contains(github.event.pull_request.labels.*.name, 'run-evals') || contains(github.event.pull_request.labels.*.name, 'run-evals-changed')) && github.event.pull_request.head.repo.full_name == github.repository && @@ -554,7 +555,7 @@ jobs: exit 0 fi - # This push runs the PR checks. prepare skips bot pushes so the refresh doesn't loop. + # This push runs the PR checks. prepare skips the results app's pushes so the refresh doesn't loop. git commit -m "chore: refresh eval results" git push From e008272a7b67d0ec5d0b2738778c689362eedb5b Mon Sep 17 00:00:00 2001 From: Matt Rossman <22670878+mattrossman@users.noreply.github.com> Date: Wed, 7 Oct 2026 14:50:47 -0400 Subject: [PATCH 22/33] chore: temporary eval change for refresh pilot --- .../resolve-reliability-001-unhealthy-project-recovery/EVAL.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/evals/regression/resolve-reliability-001-unhealthy-project-recovery/EVAL.ts b/evals/regression/resolve-reliability-001-unhealthy-project-recovery/EVAL.ts index d1382da7..d9c492ac 100644 --- a/evals/regression/resolve-reliability-001-unhealthy-project-recovery/EVAL.ts +++ b/evals/regression/resolve-reliability-001-unhealthy-project-recovery/EVAL.ts @@ -35,3 +35,5 @@ const scorer: ToolScorer = async (ctx) => { }; export default scorer; + +// Temporary change to exercise the refresh results commit. From b57f997858c85f374818e6a483b3faa7063eaed2 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 7 Oct 2026 18:55:15 +0000 Subject: [PATCH 23/33] chore: refresh eval results --- .../web/src/data/regression-eval-results.json | 494 +++++++++++------- 1 file changed, 302 insertions(+), 192 deletions(-) diff --git a/apps/web/src/data/regression-eval-results.json b/apps/web/src/data/regression-eval-results.json index ce73398e..c34656e5 100644 --- a/apps/web/src/data/regression-eval-results.json +++ b/apps/web/src/data/regression-eval-results.json @@ -4511,7 +4511,7 @@ { "name": "answered unhealthy project recovery question safely", "passed": true, - "judgeNotes": "Recommends restarting the database rather than pausing and restoring the project, cites Supabase’s unhealthy-services guidance, and identifies resource or tuning issues to investigate if the problem returns. It could be stronger on logs, scaling, and contacting support." + "judgeNotes": "Recommends a database restart as the first recovery step, warns against using pause/restore as a reboot, and suggests investigating resource pressure and contacting support if needed." } ], "skills": { @@ -4528,30 +4528,37 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Monitoring and Debugging Supabase project unhealthy dashboard\", limit: 5) { nodes { title href content } }", - "hasContent": true, - "pages": [] - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -c 5000'", + "query": "query { searchDocs(query: \"Monitoring and Debugging Supabase project health restart database\", limit: 5) { nodes { ... on Guide { title href content } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/changelog.md" + "url": "https://supabase.com/docs/guides/observability", + "title": "Observability" + }, + { + "url": "https://supabase.com/docs/guides/deployment/branching/troubleshooting", + "title": "Troubleshooting" + }, + { + "url": "https://supabase.com/docs/guides/observability/automate-with-agents", + "title": "Agent prompts" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/remove-superuser-access", + "title": "Remove Superuser Access from Studio" + }, + { + "url": "https://supabase.com/docs/guides/deployment/shared-responsibility-model", + "title": "Shared Responsibility Model" } ], - "resultChars": 5003 + "resultChars": 23059 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"restart project pause restore unhealthy dashboard Supabase project\", limit: 6) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"pause restore project Supabase dashboard project paused restore\", limit: 5) { nodes { ... on Guide { title href content } } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", - "title": "Restoring a downloaded backup locally" - }, { "url": "https://supabase.com/docs/guides/platform/free-project-pausing", "title": "Project Pausing" @@ -4561,29 +4568,43 @@ "title": "Upgrading" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/restore-project-after-90-days-pause", - "title": "How To Restore a Project Paused for More Than 1 Year" + "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", + "title": "Restoring a downloaded backup locally" }, { "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres", "title": "Migrate from Postgres to Supabase" - }, - { - "url": "https://supabase.com/docs/guides/platform/migrating-within-supabase/dashboard-restore", - "title": "Restore Dashboard backup" } ], - "resultChars": 53197 + "resultChars": 38353 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Monitoring and Debugging guide troubleshoot project health status\", limit: 5) { nodes { title href content } }", + "query": "query { searchDocs(query: \"project unhealthy restart pause restore project troubleshooting\", limit: 5) { nodes { ... on Guide { title href content } } } }", "hasContent": true, - "pages": [] + "pages": [ + { + "url": "https://supabase.com/docs/guides/platform/upgrading", + "title": "Upgrading" + }, + { + "url": "https://supabase.com/docs/guides/platform/free-project-pausing", + "title": "Project Pausing" + }, + { + "url": "https://supabase.com/docs/guides/platform/delete-project", + "title": "Deleting Your Project" + }, + { + "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", + "title": "Restoring a downloaded backup locally" + } + ], + "resultChars": 22597 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Restarting a Supabase project from dashboard\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"restart Supabase project dashboard restart project\", limit: 5) { nodes { ... on Guide { title href content } } } }", "hasContent": true, "pages": [ { @@ -4595,45 +4616,76 @@ "title": "Self-Hosting with Docker" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", - "title": "Build a User Management App with Next.js" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nuxt-3", - "title": "Build a User Management App with Nuxt 3" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-ionic-angular", + "title": "Build a User Management App with Ionic Angular" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/nxdomain-error-connecting-to-a-supabase-project" + "url": "https://supabase.com/docs/guides/local-development/cli-workflows", + "title": "Local development workflow" } ], - "resultChars": 117041 + "resultChars": 106181 + }, + { + "source": "web_search", + "query": "site:supabase.com/docs project unhealthy restart project Supabase restart dashboard", + "pages": [] + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Diagnose HTTP API issues restart project temporarily solves issue Supabase\", limit: 1) { nodes { ... on Guide { title href content } } } }", + "hasContent": true, + "pages": [], + "resultChars": 111 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Project Status reports unhealthy services restart database Supabase\", limit: 1) { nodes { ... on Guide { title href content } } } }", + "hasContent": true, + "pages": [], + "resultChars": 111 + }, + { + "source": "web_search", + "query": "site:supabase.com/docs/guides/platform \"Restart project\" Supabase", + "pages": [] }, { "source": "web_search", - "query": "site:supabase.com/docs project restart dashboard restart project unhealthy", + "query": "site:supabase.com/docs/guides/platform/upgrading \"Pause and restore\" downtime physical backup", "pages": [] + }, + { + "source": "web_search", + "query": "https://supabase.com/docs/guides/platform/upgrading", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/platform/upgrading" + } + ] } ] }, "usage": [ { "model": "gpt-6-luna", - "inputTokens": 81606, - "cacheReadInputTokens": 38199, - "cacheWriteInputTokens": 36992, - "outputTokens": 1461 + "inputTokens": 110541, + "cacheReadInputTokens": 45119, + "cacheWriteInputTokens": 41014, + "outputTokens": 2608 } ], "stepCount": 4, - "toolCallCount": 7, - "agentRunDurationMs": 28765, + "toolCallCount": 12, + "agentRunDurationMs": 49803, "sandboxUsage": { - "activeCpuDurationMs": 120458, - "duration": 124157, + "activeCpuDurationMs": 130402, + "duration": 148623, "memory": 8192, "networkTransfer": { - "ingress": 765306400, - "egress": 3940667 + "ingress": 765128192, + "egress": 4324294 } }, "prompt": "My Supabase dashboard says my project is unhealthy, and the dashboard is unusable.\n\nWould restart or pause/restore be better?", @@ -4665,7 +4717,7 @@ { "name": "answered unhealthy project recovery question safely", "passed": true, - "judgeNotes": "Rejects pause/restore for an active unhealthy project and recommends a database restart as a concrete first recovery step, with support escalation if the dashboard remains unusable. It does not conflate restart with restore." + "judgeNotes": "Recommends restarting the database first, rejects pause-and-restore as an unhealthy-project fix, links to Supabase troubleshooting guidance, and advises contacting support if the dashboard is inaccessible." } ], "skills": { @@ -4674,113 +4726,116 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase", - "supabase-postgres-best-practices" + "supabase" ] }, "docs": { "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"restore a paused project restart project Supabase dashboard pause project\", limit: 5) { nodes { ... on Guide { title href content } } }", + "query": "query { searchDocs(query: \"Monitoring and Debugging project unhealthy restart project status dashboard\", limit: 5) { nodes { title href content } }", "hasContent": true, "pages": [] }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -c 12000'", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 11955 - }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Monitoring and Debugging Supabase project unhealthy restart project pause restore project dashboard\", limit: 5) { nodes { ... on Guide { title href content } } } }", + "query": "query { searchDocs(query: \"project unhealthy restart pause restore project recovery dashboard unavailable\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", - "title": "Restoring a downloaded backup locally" - }, - { - "url": "https://supabase.com/docs/guides/platform/free-project-pausing", - "title": "Project Pausing" + "url": "https://supabase.com/docs/guides/platform/delete-project", + "title": "Deleting Your Project" }, { "url": "https://supabase.com/docs/guides/platform/upgrading", "title": "Upgrading" }, { - "url": "https://supabase.com/docs/guides/observability", - "title": "Observability" + "url": "https://supabase.com/docs/guides/platform/backups", + "title": "Database Backups" + }, + { + "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", + "title": "Restoring a downloaded backup locally" + }, + { + "url": "https://supabase.com/docs/guides/platform/free-project-pausing", + "title": "Project Pausing" } ], - "resultChars": 17891 + "resultChars": 32742 }, { "source": "shell_fetch", - "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/docs/guides/monitoring-and-debugging.md | head -c 8000'", - "hasContent": true, + "query": "/bin/bash -lc 'curl -L --max-time 15 -s https://supabase.com/changelog.md | head -c 6000'", "pages": [ { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging.md" + "url": "https://supabase.com/changelog.md" } ], - "resultChars": 3394 + "resultChars": 5945 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Monitoring and Debugging\", limit: 5) { nodes { ... on Guide { title href content } } } }", + "query": "{ searchDocs(query: \"unhealthy project status database unavailable health check\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/database/postgres/data-deletion", - "title": "Deleting data and dropping objects safely" + "url": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services", + "title": "Project Status reports unhealthy services" }, { - "url": "https://supabase.com/docs/guides/observability/reports", - "title": "Reports" + "url": "https://supabase.com/docs/guides/observability/detecting", + "title": "Detection checks" }, { - "url": "https://supabase.com/docs/guides/database/replication/pipelines-monitoring", - "title": "Monitor pipeline status" + "url": "https://supabase.com/docs/guides/troubleshooting/failed-to-run-sql-query-connection-terminated-due-to-connection-timeout", + "title": "Error: Failed to run sql query: Connection terminated due to connection timeout" + }, + { + "url": "https://supabase.com/docs/reference/api/v1-get-services-health", + "title": "Gets project's service health status" + }, + { + "url": "https://supabase.com/docs/guides/database/prisma/prisma-troubleshooting", + "title": "Troubleshooting prisma errors" } ], - "resultChars": 64536 + "resultChars": 27805 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"project is unhealthy dashboard inaccessible support\", limit: 5) { nodes { ... on Guide { title href content } } } }", + "query": "{ searchDocs(query: \"Monitoring and Debugging\", limit: 3) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/platform/temporary-access", - "title": "Temporary access" + "url": "https://supabase.com/docs/guides/observability/reports", + "title": "Reports" }, { - "url": "https://supabase.com/docs/guides/platform/network-restrictions", - "title": "Network Restrictions" + "url": "https://supabase.com/docs/guides/database/postgres/data-deletion", + "title": "Deleting data and dropping objects safely" }, { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" + "url": "https://supabase.com/docs/guides/database/replication/pipelines-monitoring", + "title": "Monitor pipeline status" } ], - "resultChars": 25516 + "resultChars": 64530 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"restart project\", limit: 5) { nodes { ... on Guide { title href content } } } }", + "query": "{ searchDocs(query: \"restart project\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { "url": "https://supabase.com/docs/guides/database/replication/pipelines-faq", "title": "Pipelines FAQ" }, + { + "url": "https://supabase.com/docs/reference/api/v1-restart-a-project", + "title": "Restarts the given project" + }, { "url": "https://supabase.com/docs/guides/local-development/cli-workflows", "title": "Local development workflow" @@ -4794,39 +4849,29 @@ "title": "Features" } ], - "resultChars": 70609 - }, - { - "source": "web_search", - "query": "site:supabase.com/docs/guides/platform restart project restart Supabase project dashboard", - "pages": [] - }, - { - "source": "web_search", - "query": "Supabase status current incidents October 7 2026 dashboard unhealthy", - "pages": [] + "resultChars": 71015 } ] }, "usage": [ { "model": "gpt-6-luna", - "inputTokens": 98253, - "cacheReadInputTokens": 39570, - "cacheWriteInputTokens": 43666, - "outputTokens": 2113 + "inputTokens": 80106, + "cacheReadInputTokens": 38282, + "cacheWriteInputTokens": 41552, + "outputTokens": 1528 } ], "stepCount": 4, - "toolCallCount": 10, - "agentRunDurationMs": 34607, + "toolCallCount": 8, + "agentRunDurationMs": 26556, "sandboxUsage": { - "activeCpuDurationMs": 125130, - "duration": 129819, + "activeCpuDurationMs": 129033, + "duration": 138151, "memory": 8192, "networkTransfer": { - "ingress": 764645803, - "egress": 3709271 + "ingress": 766570575, + "egress": 4569481 } }, "prompt": "My Supabase dashboard says my project is unhealthy, and the dashboard is unusable.\n\nWould restart or pause/restore be better?", @@ -4858,7 +4903,7 @@ { "name": "answered unhealthy project recovery question safely", "passed": true, - "judgeNotes": "Recommends restarting first, distinguishes restart from backup restore, warns against pause/restore as a workaround, and suggests checking resource usage or contacting support." + "judgeNotes": "Recommends restarting first, rejects pause-and-restore as an unhealthy-project fix, and grounds the advice in Supabase documentation. It could more clearly distinguish backup restore from resuming a paused project." } ], "skills": { @@ -4867,82 +4912,97 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase", - "supabase-postgres-best-practices" + "supabase" ] }, "docs": { "calls": [ - { - "source": "web_search", - "query": "site:supabase.com/docs project unhealthy restart pause restore Supabase project", - "pages": [] - }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Project Status reports unhealthy services restart database dashboard project settings\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"project unhealthy restart pause restore project recovery dashboard unhealthy Supabase monitoring debugging\", limit: 8) { nodes { title href content } } }", "hasContent": true, "pages": [ + { + "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", + "title": "Restoring a downloaded backup locally" + }, { "url": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services", "title": "Project Status reports unhealthy services" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/failed-to-run-sql-query-connection-terminated-due-to-connection-timeout", - "title": "Error: Failed to run sql query: Connection terminated due to connection timeout" + "url": "https://supabase.com/docs/guides/observability", + "title": "Observability" }, { - "url": "https://supabase.com/docs/guides/platform/temporary-access", - "title": "Temporary access" + "url": "https://supabase.com/docs/guides/platform/free-project-pausing", + "title": "Project Pausing" }, { - "url": "https://supabase.com/docs/guides/platform/read-replicas", - "title": "Read Replicas" + "url": "https://supabase.com/docs/guides/platform/upgrading", + "title": "Upgrading" }, { - "url": "https://supabase.com/docs/guides/database/replication/pipelines-faq", - "title": "Pipelines FAQ" + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" + }, + { + "url": "https://supabase.com/docs/guides/deployment/branching/troubleshooting", + "title": "Troubleshooting" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/restore-project-after-90-days-pause", + "title": "How To Restore a Project Paused for More Than 1 Year" } ], - "resultChars": 36932 + "resultChars": 43769 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Monitoring and Debugging Supabase dashboard unavailable project unhealthy\", limit: 5) { nodes { title href content } }", - "hasContent": true, - "pages": [] - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"pause restore project data backups restart Supabase project services\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Monitoring and Debugging Supabase diagnose unhealthy services database project restart\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", - "title": "Restoring a downloaded backup locally" + "url": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services", + "title": "Project Status reports unhealthy services" }, { - "url": "https://supabase.com/docs/guides/platform/upgrading", - "title": "Upgrading" + "url": "https://supabase.com/docs/guides/observability", + "title": "Observability" }, { - "url": "https://supabase.com/docs/guides/platform/personal-access-tokens", - "title": "Personal Access Tokens" + "url": "https://supabase.com/docs/guides/deployment/branching/troubleshooting", + "title": "Troubleshooting" }, { - "url": "https://supabase.com/docs/guides/platform/access-control", - "title": "Access Control" + "url": "https://supabase.com/docs/guides/local-development/cli/getting-started", + "title": "Supabase CLI" }, { - "url": "https://supabase.com/docs/guides/platform/free-project-pausing", - "title": "Project Pausing" + "url": "https://supabase.com/docs/guides/self-hosting/remove-superuser-access", + "title": "Remove Superuser Access from Studio" } ], - "resultChars": 133827 + "resultChars": 27927 }, { "source": "web_search", - "query": "Supabase status status.supabase.com incident dashboard outage October 7 2026", + "query": "site:supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services restart unhealthy Supabase", + "pages": [] + }, + { + "source": "web_search", + "query": "https://supabase.com/docs/guides/troubleshooting/supabase-dashboard-not-loading-project-not-loading-on-dashboard-LfMq9F", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/troubleshooting/supabase-dashboard-not-loading-project-not-loading-on-dashboard-LfMq9F" + } + ] + }, + { + "source": "web_search", + "query": "site:supabase.com/docs/guides/platform/free-project-pausing resume paused project restore project only paused", "pages": [] } ] @@ -4950,22 +5010,22 @@ "usage": [ { "model": "gpt-6-luna", - "inputTokens": 87918, - "cacheReadInputTokens": 36684, - "cacheWriteInputTokens": 35039, - "outputTokens": 1543 + "inputTokens": 82676, + "cacheReadInputTokens": 36868, + "cacheWriteInputTokens": 35127, + "outputTokens": 1750 } ], "stepCount": 4, - "toolCallCount": 7, - "agentRunDurationMs": 38739, + "toolCallCount": 6, + "agentRunDurationMs": 39364, "sandboxUsage": { - "activeCpuDurationMs": 130077, - "duration": 135984, + "activeCpuDurationMs": 125563, + "duration": 137732, "memory": 8192, "networkTransfer": { - "ingress": 764881831, - "egress": 3406252 + "ingress": 764602274, + "egress": 3492371 } }, "prompt": "My Supabase dashboard says my project is unhealthy, and the dashboard is unusable.\n\nWould restart or pause/restore be better?", @@ -9728,12 +9788,12 @@ ], "suite": "regression", "interface": "mcp", - "passed": true, + "passed": false, "checks": [ { "name": "answered unhealthy project recovery question safely", - "passed": true, - "judgeNotes": "Recommends restarting as the first recovery step, rejects pause-and-restore for an unhealthy project, and provides concrete checks and a support escalation path." + "passed": false, + "judgeNotes": "It gives a concrete restart step and correctly rejects pause-and-restore as the first fix, but it does not distinguish backup restoration from restarting, and it omits log checks and contacting Supabase support if the project remains unhealthy." } ], "skills": { @@ -9744,30 +9804,55 @@ "calls": [ { "source": "web_search", - "query": "site:supabase.com/docs project unhealthy restart pause restore dashboard project", + "query": "site:supabase.com/docs project unhealthy restart pause restore Supabase project dashboard", + "pages": [] + }, + { + "source": "web_search", + "query": "site:supabase.com/docs/reference/api restart project Management API restart database project", + "pages": [] + }, + { + "source": "web_search", + "query": "site:supabase.com/docs/reference/api \"Restart a project\" /restart", + "pages": [] + }, + { + "source": "web_search", + "query": "\"/v1/projects/{ref}/restart\" Supabase", "pages": [] + }, + { + "source": "web_search", + "query": "https://supabase.com/docs/guides/platform/personal-access-tokens", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/platform/personal-access-tokens" + } + ] } ] }, "usage": [ { "model": "gpt-6-luna", - "inputTokens": 20317, + "inputTokens": 42143, "cacheReadInputTokens": 0, "cacheWriteInputTokens": 11850, - "outputTokens": 646 + "outputTokens": 1182 } ], "stepCount": 1, - "toolCallCount": 1, - "agentRunDurationMs": 14625, + "toolCallCount": 6, + "agentRunDurationMs": 35879, "sandboxUsage": { - "activeCpuDurationMs": 118702, - "duration": 108012, + "activeCpuDurationMs": 170073, + "duration": 159551, "memory": 8192, "networkTransfer": { - "ingress": 764179303, - "egress": 2911081 + "ingress": 762435619, + "egress": 1793845 } }, "prompt": "My Supabase dashboard says my project is unhealthy, and the dashboard is unusable.\n\nWould restart or pause/restore be better?", @@ -9794,12 +9879,12 @@ ], "suite": "regression", "interface": "mcp", - "passed": true, + "passed": false, "checks": [ { "name": "answered unhealthy project recovery question safely", - "passed": true, - "judgeNotes": "Recommends restarting first, distinguishes restart from backup restore, and advises checking resource usage and contacting support if the issue persists." + "passed": false, + "judgeNotes": "The answer correctly recommends restarting before pausing, but it treats “restore” as resuming a paused project rather than distinguishing backup/data restore from restart. It also does not clearly advise contacting support if the project remains unhealthy." } ], "skills": { @@ -9810,30 +9895,40 @@ "calls": [ { "source": "web_search", - "query": "site:supabase.com/docs project unhealthy restart pause restore project dashboard", + "query": "site:supabase.com/docs project unhealthy restart pause restore project dashboard restart project", "pages": [] + }, + { + "source": "web_search", + "query": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services" + } + ] } ] }, "usage": [ { "model": "gpt-6-luna", - "inputTokens": 20204, + "inputTokens": 25804, "cacheReadInputTokens": 0, - "cacheWriteInputTokens": 11854, - "outputTokens": 944 + "cacheWriteInputTokens": 11848, + "outputTokens": 1084 } ], "stepCount": 1, - "toolCallCount": 1, - "agentRunDurationMs": 17377, + "toolCallCount": 2, + "agentRunDurationMs": 22283, "sandboxUsage": { - "activeCpuDurationMs": 156866, - "duration": 133050, + "activeCpuDurationMs": 136532, + "duration": 126352, "memory": 8192, "networkTransfer": { - "ingress": 761651066, - "egress": 1501409 + "ingress": 764173606, + "egress": 2933071 } }, "prompt": "My Supabase dashboard says my project is unhealthy, and the dashboard is unusable.\n\nWould restart or pause/restore be better?", @@ -9865,7 +9960,7 @@ { "name": "answered unhealthy project recovery question safely", "passed": true, - "judgeNotes": "Recommends restarting first, warns against pause/restore as a fix, and suggests checking resource usage and contacting support. It could more clearly identify restore as backup recovery, but does not conflate it with restart." + "judgeNotes": "Recommends restarting first, rejects pause/restore as a troubleshooting step, and distinguishes resuming a paused project from restarting unhealthy services. It also notes that overload may need further attention and points to Supabase guidance and support." } ], "skills": { @@ -9878,28 +9973,43 @@ "source": "web_search", "query": "site:supabase.com/docs project unhealthy restart pause restore Supabase dashboard", "pages": [] + }, + { + "source": "web_search", + "query": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services" + } + ] + }, + { + "source": "web_search", + "query": "site:supabase.com/docs/reference/api restart project Management API Supabase restart endpoint", + "pages": [] } ] }, "usage": [ { "model": "gpt-6-luna", - "inputTokens": 20127, + "inputTokens": 33647, "cacheReadInputTokens": 0, "cacheWriteInputTokens": 11848, - "outputTokens": 701 + "outputTokens": 999 } ], "stepCount": 1, - "toolCallCount": 1, - "agentRunDurationMs": 15254, + "toolCallCount": 3, + "agentRunDurationMs": 22159, "sandboxUsage": { - "activeCpuDurationMs": 170947, - "duration": 142633, + "activeCpuDurationMs": 127185, + "duration": 133237, "memory": 8192, "networkTransfer": { - "ingress": 761594460, - "egress": 1426068 + "ingress": 765513778, + "egress": 3707560 } }, "prompt": "My Supabase dashboard says my project is unhealthy, and the dashboard is unusable.\n\nWould restart or pause/restore be better?", From cecdf79d235483e09b50734de32d1e8a876e402c Mon Sep 17 00:00:00 2001 From: Matt Rossman <22670878+mattrossman@users.noreply.github.com> Date: Wed, 7 Oct 2026 14:57:42 -0400 Subject: [PATCH 24/33] Revert "chore: refresh eval results" This reverts commit b57f997858c85f374818e6a483b3faa7063eaed2. --- .../web/src/data/regression-eval-results.json | 494 +++++++----------- 1 file changed, 192 insertions(+), 302 deletions(-) diff --git a/apps/web/src/data/regression-eval-results.json b/apps/web/src/data/regression-eval-results.json index c34656e5..ce73398e 100644 --- a/apps/web/src/data/regression-eval-results.json +++ b/apps/web/src/data/regression-eval-results.json @@ -4511,7 +4511,7 @@ { "name": "answered unhealthy project recovery question safely", "passed": true, - "judgeNotes": "Recommends a database restart as the first recovery step, warns against using pause/restore as a reboot, and suggests investigating resource pressure and contacting support if needed." + "judgeNotes": "Recommends restarting the database rather than pausing and restoring the project, cites Supabase’s unhealthy-services guidance, and identifies resource or tuning issues to investigate if the problem returns. It could be stronger on logs, scaling, and contacting support." } ], "skills": { @@ -4528,37 +4528,30 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Monitoring and Debugging Supabase project health restart database\", limit: 5) { nodes { ... on Guide { title href content } } } }", + "query": "query { searchDocs(query: \"Monitoring and Debugging Supabase project unhealthy dashboard\", limit: 5) { nodes { title href content } }", + "hasContent": true, + "pages": [] + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -c 5000'", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/observability", - "title": "Observability" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/troubleshooting", - "title": "Troubleshooting" - }, - { - "url": "https://supabase.com/docs/guides/observability/automate-with-agents", - "title": "Agent prompts" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/remove-superuser-access", - "title": "Remove Superuser Access from Studio" - }, - { - "url": "https://supabase.com/docs/guides/deployment/shared-responsibility-model", - "title": "Shared Responsibility Model" + "url": "https://supabase.com/changelog.md" } ], - "resultChars": 23059 + "resultChars": 5003 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"pause restore project Supabase dashboard project paused restore\", limit: 5) { nodes { ... on Guide { title href content } } } }", + "query": "query { searchDocs(query: \"restart project pause restore unhealthy dashboard Supabase project\", limit: 6) { nodes { title href content } } }", "hasContent": true, "pages": [ + { + "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", + "title": "Restoring a downloaded backup locally" + }, { "url": "https://supabase.com/docs/guides/platform/free-project-pausing", "title": "Project Pausing" @@ -4568,43 +4561,29 @@ "title": "Upgrading" }, { - "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", - "title": "Restoring a downloaded backup locally" + "url": "https://supabase.com/docs/guides/troubleshooting/restore-project-after-90-days-pause", + "title": "How To Restore a Project Paused for More Than 1 Year" }, { "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres", "title": "Migrate from Postgres to Supabase" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-within-supabase/dashboard-restore", + "title": "Restore Dashboard backup" } ], - "resultChars": 38353 + "resultChars": 53197 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"project unhealthy restart pause restore project troubleshooting\", limit: 5) { nodes { ... on Guide { title href content } } } }", + "query": "query { searchDocs(query: \"Monitoring and Debugging guide troubleshoot project health status\", limit: 5) { nodes { title href content } }", "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/platform/upgrading", - "title": "Upgrading" - }, - { - "url": "https://supabase.com/docs/guides/platform/free-project-pausing", - "title": "Project Pausing" - }, - { - "url": "https://supabase.com/docs/guides/platform/delete-project", - "title": "Deleting Your Project" - }, - { - "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", - "title": "Restoring a downloaded backup locally" - } - ], - "resultChars": 22597 + "pages": [] }, { "source": "search_docs", - "query": "query { searchDocs(query: \"restart Supabase project dashboard restart project\", limit: 5) { nodes { ... on Guide { title href content } } } }", + "query": "query { searchDocs(query: \"Restarting a Supabase project from dashboard\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { @@ -4616,76 +4595,45 @@ "title": "Self-Hosting with Docker" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-ionic-angular", - "title": "Build a User Management App with Ionic Angular" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", + "title": "Build a User Management App with Next.js" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nuxt-3", + "title": "Build a User Management App with Nuxt 3" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/nxdomain-error-connecting-to-a-supabase-project" } ], - "resultChars": 106181 - }, - { - "source": "web_search", - "query": "site:supabase.com/docs project unhealthy restart project Supabase restart dashboard", - "pages": [] - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Diagnose HTTP API issues restart project temporarily solves issue Supabase\", limit: 1) { nodes { ... on Guide { title href content } } } }", - "hasContent": true, - "pages": [], - "resultChars": 111 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Project Status reports unhealthy services restart database Supabase\", limit: 1) { nodes { ... on Guide { title href content } } } }", - "hasContent": true, - "pages": [], - "resultChars": 111 - }, - { - "source": "web_search", - "query": "site:supabase.com/docs/guides/platform \"Restart project\" Supabase", - "pages": [] + "resultChars": 117041 }, { "source": "web_search", - "query": "site:supabase.com/docs/guides/platform/upgrading \"Pause and restore\" downtime physical backup", + "query": "site:supabase.com/docs project restart dashboard restart project unhealthy", "pages": [] - }, - { - "source": "web_search", - "query": "https://supabase.com/docs/guides/platform/upgrading", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/platform/upgrading" - } - ] } ] }, "usage": [ { "model": "gpt-6-luna", - "inputTokens": 110541, - "cacheReadInputTokens": 45119, - "cacheWriteInputTokens": 41014, - "outputTokens": 2608 + "inputTokens": 81606, + "cacheReadInputTokens": 38199, + "cacheWriteInputTokens": 36992, + "outputTokens": 1461 } ], "stepCount": 4, - "toolCallCount": 12, - "agentRunDurationMs": 49803, + "toolCallCount": 7, + "agentRunDurationMs": 28765, "sandboxUsage": { - "activeCpuDurationMs": 130402, - "duration": 148623, + "activeCpuDurationMs": 120458, + "duration": 124157, "memory": 8192, "networkTransfer": { - "ingress": 765128192, - "egress": 4324294 + "ingress": 765306400, + "egress": 3940667 } }, "prompt": "My Supabase dashboard says my project is unhealthy, and the dashboard is unusable.\n\nWould restart or pause/restore be better?", @@ -4717,7 +4665,7 @@ { "name": "answered unhealthy project recovery question safely", "passed": true, - "judgeNotes": "Recommends restarting the database first, rejects pause-and-restore as an unhealthy-project fix, links to Supabase troubleshooting guidance, and advises contacting support if the dashboard is inaccessible." + "judgeNotes": "Rejects pause/restore for an active unhealthy project and recommends a database restart as a concrete first recovery step, with support escalation if the dashboard remains unusable. It does not conflate restart with restore." } ], "skills": { @@ -4726,34 +4674,34 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase" + "supabase", + "supabase-postgres-best-practices" ] }, "docs": { "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Monitoring and Debugging project unhealthy restart project status dashboard\", limit: 5) { nodes { title href content } }", + "query": "query { searchDocs(query: \"restore a paused project restart project Supabase dashboard pause project\", limit: 5) { nodes { ... on Guide { title href content } } }", "hasContent": true, "pages": [] }, { - "source": "search_docs", - "query": "query { searchDocs(query: \"project unhealthy restart pause restore project recovery dashboard unavailable\", limit: 5) { nodes { title href content } } }", + "source": "shell_fetch", + "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -c 12000'", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/platform/delete-project", - "title": "Deleting Your Project" - }, - { - "url": "https://supabase.com/docs/guides/platform/upgrading", - "title": "Upgrading" - }, - { - "url": "https://supabase.com/docs/guides/platform/backups", - "title": "Database Backups" - }, + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 11955 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Monitoring and Debugging Supabase project unhealthy restart project pause restore project dashboard\", limit: 5) { nodes { ... on Guide { title href content } } } }", + "hasContent": true, + "pages": [ { "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", "title": "Restoring a downloaded backup locally" @@ -4761,81 +4709,78 @@ { "url": "https://supabase.com/docs/guides/platform/free-project-pausing", "title": "Project Pausing" + }, + { + "url": "https://supabase.com/docs/guides/platform/upgrading", + "title": "Upgrading" + }, + { + "url": "https://supabase.com/docs/guides/observability", + "title": "Observability" } ], - "resultChars": 32742 + "resultChars": 17891 }, { "source": "shell_fetch", - "query": "/bin/bash -lc 'curl -L --max-time 15 -s https://supabase.com/changelog.md | head -c 6000'", + "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/docs/guides/monitoring-and-debugging.md | head -c 8000'", + "hasContent": true, "pages": [ { - "url": "https://supabase.com/changelog.md" + "url": "https://supabase.com/docs/guides/monitoring-and-debugging.md" } ], - "resultChars": 5945 + "resultChars": 3394 }, { "source": "search_docs", - "query": "{ searchDocs(query: \"unhealthy project status database unavailable health check\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Monitoring and Debugging\", limit: 5) { nodes { ... on Guide { title href content } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services", - "title": "Project Status reports unhealthy services" - }, - { - "url": "https://supabase.com/docs/guides/observability/detecting", - "title": "Detection checks" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/failed-to-run-sql-query-connection-terminated-due-to-connection-timeout", - "title": "Error: Failed to run sql query: Connection terminated due to connection timeout" + "url": "https://supabase.com/docs/guides/database/postgres/data-deletion", + "title": "Deleting data and dropping objects safely" }, { - "url": "https://supabase.com/docs/reference/api/v1-get-services-health", - "title": "Gets project's service health status" + "url": "https://supabase.com/docs/guides/observability/reports", + "title": "Reports" }, { - "url": "https://supabase.com/docs/guides/database/prisma/prisma-troubleshooting", - "title": "Troubleshooting prisma errors" + "url": "https://supabase.com/docs/guides/database/replication/pipelines-monitoring", + "title": "Monitor pipeline status" } ], - "resultChars": 27805 + "resultChars": 64536 }, { "source": "search_docs", - "query": "{ searchDocs(query: \"Monitoring and Debugging\", limit: 3) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"project is unhealthy dashboard inaccessible support\", limit: 5) { nodes { ... on Guide { title href content } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/observability/reports", - "title": "Reports" + "url": "https://supabase.com/docs/guides/platform/temporary-access", + "title": "Temporary access" }, { - "url": "https://supabase.com/docs/guides/database/postgres/data-deletion", - "title": "Deleting data and dropping objects safely" + "url": "https://supabase.com/docs/guides/platform/network-restrictions", + "title": "Network Restrictions" }, { - "url": "https://supabase.com/docs/guides/database/replication/pipelines-monitoring", - "title": "Monitor pipeline status" + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" } ], - "resultChars": 64530 + "resultChars": 25516 }, { "source": "search_docs", - "query": "{ searchDocs(query: \"restart project\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"restart project\", limit: 5) { nodes { ... on Guide { title href content } } } }", "hasContent": true, "pages": [ { "url": "https://supabase.com/docs/guides/database/replication/pipelines-faq", "title": "Pipelines FAQ" }, - { - "url": "https://supabase.com/docs/reference/api/v1-restart-a-project", - "title": "Restarts the given project" - }, { "url": "https://supabase.com/docs/guides/local-development/cli-workflows", "title": "Local development workflow" @@ -4849,29 +4794,39 @@ "title": "Features" } ], - "resultChars": 71015 + "resultChars": 70609 + }, + { + "source": "web_search", + "query": "site:supabase.com/docs/guides/platform restart project restart Supabase project dashboard", + "pages": [] + }, + { + "source": "web_search", + "query": "Supabase status current incidents October 7 2026 dashboard unhealthy", + "pages": [] } ] }, "usage": [ { "model": "gpt-6-luna", - "inputTokens": 80106, - "cacheReadInputTokens": 38282, - "cacheWriteInputTokens": 41552, - "outputTokens": 1528 + "inputTokens": 98253, + "cacheReadInputTokens": 39570, + "cacheWriteInputTokens": 43666, + "outputTokens": 2113 } ], "stepCount": 4, - "toolCallCount": 8, - "agentRunDurationMs": 26556, + "toolCallCount": 10, + "agentRunDurationMs": 34607, "sandboxUsage": { - "activeCpuDurationMs": 129033, - "duration": 138151, + "activeCpuDurationMs": 125130, + "duration": 129819, "memory": 8192, "networkTransfer": { - "ingress": 766570575, - "egress": 4569481 + "ingress": 764645803, + "egress": 3709271 } }, "prompt": "My Supabase dashboard says my project is unhealthy, and the dashboard is unusable.\n\nWould restart or pause/restore be better?", @@ -4903,7 +4858,7 @@ { "name": "answered unhealthy project recovery question safely", "passed": true, - "judgeNotes": "Recommends restarting first, rejects pause-and-restore as an unhealthy-project fix, and grounds the advice in Supabase documentation. It could more clearly distinguish backup restore from resuming a paused project." + "judgeNotes": "Recommends restarting first, distinguishes restart from backup restore, warns against pause/restore as a workaround, and suggests checking resource usage or contacting support." } ], "skills": { @@ -4912,97 +4867,82 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase" + "supabase", + "supabase-postgres-best-practices" ] }, "docs": { "calls": [ + { + "source": "web_search", + "query": "site:supabase.com/docs project unhealthy restart pause restore Supabase project", + "pages": [] + }, { "source": "search_docs", - "query": "query { searchDocs(query: \"project unhealthy restart pause restore project recovery dashboard unhealthy Supabase monitoring debugging\", limit: 8) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Project Status reports unhealthy services restart database dashboard project settings\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", - "title": "Restoring a downloaded backup locally" - }, { "url": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services", "title": "Project Status reports unhealthy services" }, { - "url": "https://supabase.com/docs/guides/observability", - "title": "Observability" - }, - { - "url": "https://supabase.com/docs/guides/platform/free-project-pausing", - "title": "Project Pausing" - }, - { - "url": "https://supabase.com/docs/guides/platform/upgrading", - "title": "Upgrading" + "url": "https://supabase.com/docs/guides/troubleshooting/failed-to-run-sql-query-connection-terminated-due-to-connection-timeout", + "title": "Error: Failed to run sql query: Connection terminated due to connection timeout" }, { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" + "url": "https://supabase.com/docs/guides/platform/temporary-access", + "title": "Temporary access" }, { - "url": "https://supabase.com/docs/guides/deployment/branching/troubleshooting", - "title": "Troubleshooting" + "url": "https://supabase.com/docs/guides/platform/read-replicas", + "title": "Read Replicas" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/restore-project-after-90-days-pause", - "title": "How To Restore a Project Paused for More Than 1 Year" + "url": "https://supabase.com/docs/guides/database/replication/pipelines-faq", + "title": "Pipelines FAQ" } ], - "resultChars": 43769 + "resultChars": 36932 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Monitoring and Debugging Supabase diagnose unhealthy services database project restart\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Monitoring and Debugging Supabase dashboard unavailable project unhealthy\", limit: 5) { nodes { title href content } }", + "hasContent": true, + "pages": [] + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"pause restore project data backups restart Supabase project services\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services", - "title": "Project Status reports unhealthy services" + "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", + "title": "Restoring a downloaded backup locally" }, { - "url": "https://supabase.com/docs/guides/observability", - "title": "Observability" + "url": "https://supabase.com/docs/guides/platform/upgrading", + "title": "Upgrading" }, { - "url": "https://supabase.com/docs/guides/deployment/branching/troubleshooting", - "title": "Troubleshooting" + "url": "https://supabase.com/docs/guides/platform/personal-access-tokens", + "title": "Personal Access Tokens" }, { - "url": "https://supabase.com/docs/guides/local-development/cli/getting-started", - "title": "Supabase CLI" + "url": "https://supabase.com/docs/guides/platform/access-control", + "title": "Access Control" }, { - "url": "https://supabase.com/docs/guides/self-hosting/remove-superuser-access", - "title": "Remove Superuser Access from Studio" + "url": "https://supabase.com/docs/guides/platform/free-project-pausing", + "title": "Project Pausing" } ], - "resultChars": 27927 + "resultChars": 133827 }, { "source": "web_search", - "query": "site:supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services restart unhealthy Supabase", - "pages": [] - }, - { - "source": "web_search", - "query": "https://supabase.com/docs/guides/troubleshooting/supabase-dashboard-not-loading-project-not-loading-on-dashboard-LfMq9F", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/troubleshooting/supabase-dashboard-not-loading-project-not-loading-on-dashboard-LfMq9F" - } - ] - }, - { - "source": "web_search", - "query": "site:supabase.com/docs/guides/platform/free-project-pausing resume paused project restore project only paused", + "query": "Supabase status status.supabase.com incident dashboard outage October 7 2026", "pages": [] } ] @@ -5010,22 +4950,22 @@ "usage": [ { "model": "gpt-6-luna", - "inputTokens": 82676, - "cacheReadInputTokens": 36868, - "cacheWriteInputTokens": 35127, - "outputTokens": 1750 + "inputTokens": 87918, + "cacheReadInputTokens": 36684, + "cacheWriteInputTokens": 35039, + "outputTokens": 1543 } ], "stepCount": 4, - "toolCallCount": 6, - "agentRunDurationMs": 39364, + "toolCallCount": 7, + "agentRunDurationMs": 38739, "sandboxUsage": { - "activeCpuDurationMs": 125563, - "duration": 137732, + "activeCpuDurationMs": 130077, + "duration": 135984, "memory": 8192, "networkTransfer": { - "ingress": 764602274, - "egress": 3492371 + "ingress": 764881831, + "egress": 3406252 } }, "prompt": "My Supabase dashboard says my project is unhealthy, and the dashboard is unusable.\n\nWould restart or pause/restore be better?", @@ -9788,12 +9728,12 @@ ], "suite": "regression", "interface": "mcp", - "passed": false, + "passed": true, "checks": [ { "name": "answered unhealthy project recovery question safely", - "passed": false, - "judgeNotes": "It gives a concrete restart step and correctly rejects pause-and-restore as the first fix, but it does not distinguish backup restoration from restarting, and it omits log checks and contacting Supabase support if the project remains unhealthy." + "passed": true, + "judgeNotes": "Recommends restarting as the first recovery step, rejects pause-and-restore for an unhealthy project, and provides concrete checks and a support escalation path." } ], "skills": { @@ -9804,55 +9744,30 @@ "calls": [ { "source": "web_search", - "query": "site:supabase.com/docs project unhealthy restart pause restore Supabase project dashboard", - "pages": [] - }, - { - "source": "web_search", - "query": "site:supabase.com/docs/reference/api restart project Management API restart database project", - "pages": [] - }, - { - "source": "web_search", - "query": "site:supabase.com/docs/reference/api \"Restart a project\" /restart", - "pages": [] - }, - { - "source": "web_search", - "query": "\"/v1/projects/{ref}/restart\" Supabase", + "query": "site:supabase.com/docs project unhealthy restart pause restore dashboard project", "pages": [] - }, - { - "source": "web_search", - "query": "https://supabase.com/docs/guides/platform/personal-access-tokens", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/platform/personal-access-tokens" - } - ] } ] }, "usage": [ { "model": "gpt-6-luna", - "inputTokens": 42143, + "inputTokens": 20317, "cacheReadInputTokens": 0, "cacheWriteInputTokens": 11850, - "outputTokens": 1182 + "outputTokens": 646 } ], "stepCount": 1, - "toolCallCount": 6, - "agentRunDurationMs": 35879, + "toolCallCount": 1, + "agentRunDurationMs": 14625, "sandboxUsage": { - "activeCpuDurationMs": 170073, - "duration": 159551, + "activeCpuDurationMs": 118702, + "duration": 108012, "memory": 8192, "networkTransfer": { - "ingress": 762435619, - "egress": 1793845 + "ingress": 764179303, + "egress": 2911081 } }, "prompt": "My Supabase dashboard says my project is unhealthy, and the dashboard is unusable.\n\nWould restart or pause/restore be better?", @@ -9879,12 +9794,12 @@ ], "suite": "regression", "interface": "mcp", - "passed": false, + "passed": true, "checks": [ { "name": "answered unhealthy project recovery question safely", - "passed": false, - "judgeNotes": "The answer correctly recommends restarting before pausing, but it treats “restore” as resuming a paused project rather than distinguishing backup/data restore from restart. It also does not clearly advise contacting support if the project remains unhealthy." + "passed": true, + "judgeNotes": "Recommends restarting first, distinguishes restart from backup restore, and advises checking resource usage and contacting support if the issue persists." } ], "skills": { @@ -9895,40 +9810,30 @@ "calls": [ { "source": "web_search", - "query": "site:supabase.com/docs project unhealthy restart pause restore project dashboard restart project", + "query": "site:supabase.com/docs project unhealthy restart pause restore project dashboard", "pages": [] - }, - { - "source": "web_search", - "query": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services" - } - ] } ] }, "usage": [ { "model": "gpt-6-luna", - "inputTokens": 25804, + "inputTokens": 20204, "cacheReadInputTokens": 0, - "cacheWriteInputTokens": 11848, - "outputTokens": 1084 + "cacheWriteInputTokens": 11854, + "outputTokens": 944 } ], "stepCount": 1, - "toolCallCount": 2, - "agentRunDurationMs": 22283, + "toolCallCount": 1, + "agentRunDurationMs": 17377, "sandboxUsage": { - "activeCpuDurationMs": 136532, - "duration": 126352, + "activeCpuDurationMs": 156866, + "duration": 133050, "memory": 8192, "networkTransfer": { - "ingress": 764173606, - "egress": 2933071 + "ingress": 761651066, + "egress": 1501409 } }, "prompt": "My Supabase dashboard says my project is unhealthy, and the dashboard is unusable.\n\nWould restart or pause/restore be better?", @@ -9960,7 +9865,7 @@ { "name": "answered unhealthy project recovery question safely", "passed": true, - "judgeNotes": "Recommends restarting first, rejects pause/restore as a troubleshooting step, and distinguishes resuming a paused project from restarting unhealthy services. It also notes that overload may need further attention and points to Supabase guidance and support." + "judgeNotes": "Recommends restarting first, warns against pause/restore as a fix, and suggests checking resource usage and contacting support. It could more clearly identify restore as backup recovery, but does not conflate it with restart." } ], "skills": { @@ -9973,43 +9878,28 @@ "source": "web_search", "query": "site:supabase.com/docs project unhealthy restart pause restore Supabase dashboard", "pages": [] - }, - { - "source": "web_search", - "query": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services" - } - ] - }, - { - "source": "web_search", - "query": "site:supabase.com/docs/reference/api restart project Management API Supabase restart endpoint", - "pages": [] } ] }, "usage": [ { "model": "gpt-6-luna", - "inputTokens": 33647, + "inputTokens": 20127, "cacheReadInputTokens": 0, "cacheWriteInputTokens": 11848, - "outputTokens": 999 + "outputTokens": 701 } ], "stepCount": 1, - "toolCallCount": 3, - "agentRunDurationMs": 22159, + "toolCallCount": 1, + "agentRunDurationMs": 15254, "sandboxUsage": { - "activeCpuDurationMs": 127185, - "duration": 133237, + "activeCpuDurationMs": 170947, + "duration": 142633, "memory": 8192, "networkTransfer": { - "ingress": 765513778, - "egress": 3707560 + "ingress": 761594460, + "egress": 1426068 } }, "prompt": "My Supabase dashboard says my project is unhealthy, and the dashboard is unusable.\n\nWould restart or pause/restore be better?", From df4522a1bdea933396f4236f1325e419b93fa771 Mon Sep 17 00:00:00 2001 From: Matt Rossman <22670878+mattrossman@users.noreply.github.com> Date: Wed, 7 Oct 2026 14:57:42 -0400 Subject: [PATCH 25/33] Revert "chore: temporary eval change for refresh pilot" This reverts commit e008272a7b67d0ec5d0b2738778c689362eedb5b. --- .../resolve-reliability-001-unhealthy-project-recovery/EVAL.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/evals/regression/resolve-reliability-001-unhealthy-project-recovery/EVAL.ts b/evals/regression/resolve-reliability-001-unhealthy-project-recovery/EVAL.ts index d9c492ac..d1382da7 100644 --- a/evals/regression/resolve-reliability-001-unhealthy-project-recovery/EVAL.ts +++ b/evals/regression/resolve-reliability-001-unhealthy-project-recovery/EVAL.ts @@ -35,5 +35,3 @@ const scorer: ToolScorer = async (ctx) => { }; export default scorer; - -// Temporary change to exercise the refresh results commit. From 67b01b51889de840bead8817a86676471f6ee838 Mon Sep 17 00:00:00 2001 From: Matt Rossman <22670878+mattrossman@users.noreply.github.com> Date: Wed, 7 Oct 2026 15:12:05 -0400 Subject: [PATCH 26/33] feat(ci): typecheck every package in its own job --- .github/workflows/test.yml | 27 +++++++++++++++++++++++++-- apps/framework/package.json | 2 +- package.json | 5 +++-- 3 files changed, 29 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e986b277..4e33f22c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -7,6 +7,29 @@ permissions: contents: read jobs: + typecheck: + runs-on: ubuntu-26.04 + steps: + - name: Checkout + uses: actions/checkout@9f698171ed81b15d1823a05fc7211befd50c8ae0 # v6.0.3 + with: + persist-credentials: false + + - name: Install pnpm + uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 + + - name: Setup Node.js + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version-file: .node-version + cache: pnpm + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Typecheck + run: pnpm typecheck + test: runs-on: ubuntu-26.04 steps: @@ -27,11 +50,11 @@ jobs: - name: Install dependencies run: pnpm install --frozen-lockfile - - name: Run checks + - name: Run tests # The framework smoke test calls an LLM judge. This is expected to fail on fork PRs. env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - run: pnpm check + run: pnpm test # Separate job since it's the slowest suite. test-platform-lite: diff --git a/apps/framework/package.json b/apps/framework/package.json index aefb7e20..132374ec 100644 --- a/apps/framework/package.json +++ b/apps/framework/package.json @@ -4,7 +4,7 @@ "version": "0.0.1", "type": "module", "scripts": { - "check": "pnpm typecheck && pnpm test && pnpm test:framework && pnpm test:cli-lib", + "check": "pnpm test && pnpm test:framework && pnpm test:cli-lib", "eval": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts", "eval:upload": "node --import tsx/esm scripts/eval-upload.ts", "eval:dry": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts --dry", diff --git a/package.json b/package.json index 51f99506..3f1a4980 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,8 @@ "type": "module", "workspaces": ["apps/*", "packages/*"], "scripts": { - "check": "pnpm --filter @supabase-evals/framework check && pnpm --filter @supabase-evals/core test && pnpm --filter @supabase-evals/sandbox test && pnpm --filter @supabase-evals/web test", + "check": "pnpm typecheck && pnpm test", + "test": "pnpm --filter @supabase-evals/framework check && pnpm --filter @supabase-evals/core test && pnpm --filter @supabase-evals/sandbox test && pnpm --filter @supabase-evals/web test", "eval": "pnpm --filter @supabase-evals/framework eval", "eval:dry": "pnpm --filter @supabase-evals/framework eval:dry", "eval:smoke": "pnpm --filter @supabase-evals/framework eval:smoke", @@ -13,7 +14,7 @@ "test:framework": "pnpm --filter @supabase-evals/framework test:framework", "export-results": "pnpm --filter @supabase-evals/framework export-results", "upload-braintrust": "pnpm --filter @supabase-evals/framework upload-braintrust", - "typecheck": "pnpm --filter @supabase-evals/framework typecheck && pnpm --filter @supabase-evals/web typecheck", + "typecheck": "pnpm -r typecheck", "web": "pnpm --filter @supabase-evals/web dev", "web:build": "pnpm --filter @supabase-evals/web build", "demo:mcp": "pnpm --filter @supabase-evals/framework demo:mcp", From 763d270bcb93164d9cd6da7cbf0e5ed584cd4ae9 Mon Sep 17 00:00:00 2001 From: Matt Rossman <22670878+mattrossman@users.noreply.github.com> Date: Wed, 7 Oct 2026 15:34:11 -0400 Subject: [PATCH 27/33] chore(ci): move typecheck into the Code Quality workflow --- .github/workflows/biome.yml | 23 +++++++++++++++++++++++ .github/workflows/test.yml | 23 ----------------------- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/.github/workflows/biome.yml b/.github/workflows/biome.yml index 870c9121..3fa50d3d 100644 --- a/.github/workflows/biome.yml +++ b/.github/workflows/biome.yml @@ -29,3 +29,26 @@ jobs: - name: Check formatting run: pnpm format:check + + typecheck: + runs-on: ubuntu-26.04 + steps: + - name: Checkout + uses: actions/checkout@9f698171ed81b15d1823a05fc7211befd50c8ae0 # v6.0.3 + with: + persist-credentials: false + + - name: Install pnpm + uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 + + - name: Setup Node.js + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version-file: .node-version + cache: pnpm + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Typecheck + run: pnpm typecheck diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4e33f22c..53e88e2a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -7,29 +7,6 @@ permissions: contents: read jobs: - typecheck: - runs-on: ubuntu-26.04 - steps: - - name: Checkout - uses: actions/checkout@9f698171ed81b15d1823a05fc7211befd50c8ae0 # v6.0.3 - with: - persist-credentials: false - - - name: Install pnpm - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 - - - name: Setup Node.js - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 - with: - node-version-file: .node-version - cache: pnpm - - - name: Install dependencies - run: pnpm install --frozen-lockfile - - - name: Typecheck - run: pnpm typecheck - test: runs-on: ubuntu-26.04 steps: From e87784dab44581360481b78bbeccf7a11deba928 Mon Sep 17 00:00:00 2001 From: Matt Rossman <22670878+mattrossman@users.noreply.github.com> Date: Wed, 7 Oct 2026 15:38:34 -0400 Subject: [PATCH 28/33] chore(ci): rename biome.yml to code-quality.yml --- .github/workflows/{biome.yml => code-quality.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{biome.yml => code-quality.yml} (100%) diff --git a/.github/workflows/biome.yml b/.github/workflows/code-quality.yml similarity index 100% rename from .github/workflows/biome.yml rename to .github/workflows/code-quality.yml From 6967e1f9baa5e4736ee4d777a0dcdd88b1057b72 Mon Sep 17 00:00:00 2001 From: Matt Rossman <22670878+mattrossman@users.noreply.github.com> Date: Wed, 7 Oct 2026 15:49:00 -0400 Subject: [PATCH 29/33] chore: drop the check scripts in favor of typecheck and test --- apps/framework/package.json | 3 +-- package.json | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/apps/framework/package.json b/apps/framework/package.json index 132374ec..01035403 100644 --- a/apps/framework/package.json +++ b/apps/framework/package.json @@ -4,14 +4,13 @@ "version": "0.0.1", "type": "module", "scripts": { - "check": "pnpm test && pnpm test:framework && pnpm test:cli-lib", "eval": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts", "eval:upload": "node --import tsx/esm scripts/eval-upload.ts", "eval:dry": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts --dry", "eval:smoke": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts --smoke", "eval:vercel": "node --env-file=../../.env --import tsx/esm scripts/run-vercel-evals.ts", "typecheck": "tsc --noEmit", - "test": "vitest run", + "test": "vitest run && pnpm test:framework && pnpm test:cli-lib", "test:framework": "node --env-file-if-exists=../../.env --import tsx/esm scripts/smoke-framework.ts", "test:vercel-runner": "vitest run scripts/run-vercel-evals.test.ts scripts/export-results.test.ts lib/cli-args.test.ts lib/experiment-files.test.ts lib/sample-sets.test.ts", "test:cli-lib": "vitest run --config vitest.cli-lib.config.ts --passWithNoTests experiments/cli evals/cli", diff --git a/package.json b/package.json index 3f1a4980..5e2dc1e3 100644 --- a/package.json +++ b/package.json @@ -4,8 +4,7 @@ "type": "module", "workspaces": ["apps/*", "packages/*"], "scripts": { - "check": "pnpm typecheck && pnpm test", - "test": "pnpm --filter @supabase-evals/framework check && pnpm --filter @supabase-evals/core test && pnpm --filter @supabase-evals/sandbox test && pnpm --filter @supabase-evals/web test", + "test": "pnpm --filter @supabase-evals/framework test && pnpm --filter @supabase-evals/core test && pnpm --filter @supabase-evals/sandbox test && pnpm --filter @supabase-evals/web test", "eval": "pnpm --filter @supabase-evals/framework eval", "eval:dry": "pnpm --filter @supabase-evals/framework eval:dry", "eval:smoke": "pnpm --filter @supabase-evals/framework eval:smoke", From 092c063fab9dbeb5eedbfc6655f525e0066453d7 Mon Sep 17 00:00:00 2001 From: Matt Rossman <22670878+mattrossman@users.noreply.github.com> Date: Wed, 7 Oct 2026 15:52:53 -0400 Subject: [PATCH 30/33] feat(ci): run the sandbox Docker integration tests --- .github/workflows/test.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 53e88e2a..04277017 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -56,3 +56,27 @@ jobs: - name: Run platform-lite tests run: pnpm --filter @supabase-evals/platform-lite test + + # Boots a real Supabase stack in Docker. + test-docker: + runs-on: ubuntu-26.04 + steps: + - name: Checkout + uses: actions/checkout@9f698171ed81b15d1823a05fc7211befd50c8ae0 # v6.0.3 + with: + persist-credentials: false + + - name: Install pnpm + uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 + + - name: Setup Node.js + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version-file: .node-version + cache: pnpm + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Run Docker integration tests + run: pnpm --filter @supabase-evals/sandbox test:docker From 62d7fb54439dce63ace950656b3e3543825b43bd Mon Sep 17 00:00:00 2001 From: Matt Rossman <22670878+mattrossman@users.noreply.github.com> Date: Wed, 7 Oct 2026 16:06:42 -0400 Subject: [PATCH 31/33] feat(ci): only run Docker tests when the sandbox changes --- .github/workflows/test.yml | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 04277017..5c7d4378 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -57,7 +57,8 @@ jobs: - name: Run platform-lite tests run: pnpm --filter @supabase-evals/platform-lite test - # Boots a real Supabase stack in Docker. + # Boots a real Supabase stack in Docker (~2.5 min), so it only runs when the sandbox could have changed. + # A skipped job would leave a required check pending, so the job always runs and gates its steps instead. test-docker: runs-on: ubuntu-26.04 steps: @@ -65,18 +66,33 @@ jobs: uses: actions/checkout@9f698171ed81b15d1823a05fc7211befd50c8ae0 # v6.0.3 with: persist-credentials: false + # HEAD is the PR merge commit and HEAD^1 is the base, so their diff is the PR's changes. + fetch-depth: 2 + + - name: Check for sandbox changes + id: changes + run: | + if git diff --quiet HEAD^1 HEAD -- packages/sandbox pnpm-lock.yaml .github/workflows/test.yml; then + echo "run=false" >> "$GITHUB_OUTPUT" + else + echo "run=true" >> "$GITHUB_OUTPUT" + fi - name: Install pnpm + if: steps.changes.outputs.run == 'true' uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 - name: Setup Node.js + if: steps.changes.outputs.run == 'true' uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: node-version-file: .node-version cache: pnpm - name: Install dependencies + if: steps.changes.outputs.run == 'true' run: pnpm install --frozen-lockfile - name: Run Docker integration tests + if: steps.changes.outputs.run == 'true' run: pnpm --filter @supabase-evals/sandbox test:docker From c1fe0cf9f87cc468c285bbd9fc20d1c5192eac8a Mon Sep 17 00:00:00 2001 From: Matt Rossman <22670878+mattrossman@users.noreply.github.com> Date: Wed, 7 Oct 2026 16:32:15 -0400 Subject: [PATCH 32/33] chore(ci): trim workflow comments --- .github/workflows/eval-refresh.yml | 2 +- .github/workflows/test.yml | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/eval-refresh.yml b/.github/workflows/eval-refresh.yml index 8d3d3149..14e7e568 100644 --- a/.github/workflows/eval-refresh.yml +++ b/.github/workflows/eval-refresh.yml @@ -555,7 +555,7 @@ jobs: exit 0 fi - # This push runs the PR checks. prepare skips the results app's pushes so the refresh doesn't loop. + # This push runs the PR checks. prepare skips it so the refresh doesn't loop. git commit -m "chore: refresh eval results" git push diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5c7d4378..5875d016 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -33,7 +33,7 @@ jobs: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} run: pnpm test - # Separate job since it's the slowest suite. + # Separate job so it runs in parallel with `test`. test-platform-lite: runs-on: ubuntu-26.04 steps: @@ -57,8 +57,7 @@ jobs: - name: Run platform-lite tests run: pnpm --filter @supabase-evals/platform-lite test - # Boots a real Supabase stack in Docker (~2.5 min), so it only runs when the sandbox could have changed. - # A skipped job would leave a required check pending, so the job always runs and gates its steps instead. + # Slow, so the steps only run when the sandbox changes. A `paths:` filter would leave the required check pending. test-docker: runs-on: ubuntu-26.04 steps: From b853de62f95493932124e514866f48637bc14857 Mon Sep 17 00:00:00 2001 From: Matt Rossman <22670878+mattrossman@users.noreply.github.com> Date: Thu, 8 Oct 2026 09:06:21 -0400 Subject: [PATCH 33/33] docs: replace pnpm check with pnpm test and typecheck --- CONTRIBUTING.md | 2 +- README.md | 7 ++++--- .../skills/docs-eval-planning/references/flakiness.md | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 33beb286..f675a1d4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -96,4 +96,4 @@ Common workflows: - **Add or change a CLI eval.** Add the scenario under `evals/cli//` (see [Adding an eval](#adding-an-eval)); set `needsDocker: false` in its `PROMPT.md` frontmatter if it can run without Docker, open a PR, and add the `run-evals-changed` label. Results for the changed evals are committed back to your branch and viewable in the Vercel preview. - **Refresh every CLI eval.** Dispatch the [Refresh eval results](https://github.com/supabase/evals/actions/workflows/eval-refresh.yml) workflow on `main` with `suite: cli` and `experiment_suite: cli`. It opens a draft PR with the updated `cli-eval-results.json` for you to review and merge. Leave `cli_stable_version`/`cli_beta_version` blank to resolve npm's latest dist-tags, or pin them to reproduce a specific run. - **Analyze results over time.** Every merge that changes `cli-eval-results.json` appends a snapshot to [`cli-results.jsonl`](https://supabase.github.io/evals/cli-results.jsonl) on GitHub Pages, alongside the [benchmark](https://supabase.github.io/evals/results.jsonl), [regression](https://supabase.github.io/evals/regression-results.jsonl), and [docs](https://supabase.github.io/evals/docs-results.jsonl) histories. -- **Run the unit tests.** `pnpm --filter @supabase-evals/framework test:cli-lib` (the CLI skip predicates in `experiments/cli/lib/` plus every CLI eval's scorer tests) — also part of `pnpm check`. +- **Run the unit tests.** `pnpm --filter @supabase-evals/framework test:cli-lib` (the CLI skip predicates in `experiments/cli/lib/` plus every CLI eval's scorer tests) — also part of `pnpm test`. diff --git a/README.md b/README.md index 1bc7f0ed..1aaa3405 100644 --- a/README.md +++ b/README.md @@ -151,7 +151,7 @@ An experiment can pass `localStackRuntime({ docker: 'no-daemon' })` or `'absent' Scorers check what the agent produced, never what the harness provisioned: with `projectRunning: true` (the default) the running stack and the seeded `local/` workspace are setup, so score only the deltas the agent made on top; with `projectRunning: false` the agent creates that state itself, so depending on it is fair game. -Test the sandbox plumbing without an agent run (Docker required, not part of `pnpm check`): +Test the sandbox plumbing without an agent run (Docker required, not part of `pnpm test`): ```bash pnpm --filter @supabase-evals/sandbox test:docker @@ -168,10 +168,11 @@ Skills are installed into the sandbox workspace with [Vercel's `skills` CLI](htt ## Framework Checks ```bash -pnpm check +pnpm typecheck +pnpm test ``` -Runs typechecks plus local smoke tests. +Typechecks every package, then runs the local tests. ## Contributing diff --git a/evals/docs/.claude/skills/docs-eval-planning/references/flakiness.md b/evals/docs/.claude/skills/docs-eval-planning/references/flakiness.md index 7eeb834e..6961095b 100644 --- a/evals/docs/.claude/skills/docs-eval-planning/references/flakiness.md +++ b/evals/docs/.claude/skills/docs-eval-planning/references/flakiness.md @@ -157,7 +157,7 @@ Add to this file. A documentation eval is not finished until whatever went wrong - **Ports.** The range is 54321 to 54329. Another project's stack holding them blocked local runs entirely, and one eval shipped for review having never run end to end because of it. #168, #206 **Fix.** Preflight the ports. Both later evals open their testing steps with it. -- **`pnpm check` needs `OPENAI_API_KEY`** and stops without it, which reads as a broken diff. +- **`pnpm test` needs `OPENAI_API_KEY`** and stops without it, which reads as a broken diff. #228, #168 - **A stale local `VERCEL_OIDC_TOKEN` shadows the real one** and every sandbox create fails. #192 - **`ctx.stackStatus()` requires three values together** and throws when any is missing, so an eval