From 1a4b8892884f906381f272487a2bae38ea841c15 Mon Sep 17 00:00:00 2001 From: Colum Ferry Date: Tue, 22 Sep 2026 10:16:41 +0100 Subject: [PATCH 1/6] feat(sandbox): resolve npm channel tags for the local-stack CLI version Moves the CLI channel resolver into @supabase-evals/sandbox and lets localStackRuntime's cliVersion accept 'stable'/'beta' in addition to an exact version, resolved against npm's dist-tags at session start. An eval's own cliVersion: frontmatter pin still wins. Forwards the resolved channel pins to Vercel sandbox jobs so a run scores against one version instead of each job re-resolving "latest" independently. --- README.md | 2 + .../scripts/run-vercel-evals.test.ts | 49 ++- apps/framework/scripts/run-vercel-evals.ts | 13 +- packages/sandbox/package.json | 2 +- packages/sandbox/src/cli-channel.ts | 228 ++++++++++++ packages/sandbox/src/index.ts | 2 + packages/sandbox/src/local-stack-runtime.ts | 15 +- packages/sandbox/test/cli-channel.test.ts | 352 ++++++++++++++++++ 8 files changed, 654 insertions(+), 9 deletions(-) create mode 100644 packages/sandbox/src/cli-channel.ts create mode 100644 packages/sandbox/test/cli-channel.test.ts diff --git a/README.md b/README.md index 52a84349..adbaf62a 100644 --- a/README.md +++ b/README.md @@ -145,6 +145,8 @@ An eval's optional `local/` directory is copied into the sandbox workspace befor Set `cliVersion: 2.109.1` in an eval's frontmatter when it requires a specific Supabase CLI release. This overrides an experiment's `localStackRuntime({ cliVersion })` setting; otherwise the runtime setting or repository-wide default applies. +An experiment can instead pass `localStackRuntime({ cliVersion: 'stable' })` or `'beta'` to track npm's dist-tag for the `supabase` package rather than an exact version, resolved once at session start. An eval's own `cliVersion:` pin still wins over either form. + Scorers check what the agent produced, never what the harness provisioned: with `projectRunning: true` (the default) the running stack and the seeded `local/` workspace are setup, so score only the deltas the agent made on top; with `projectRunning: false` the agent creates that state itself, so depending on it is fair game. Test the sandbox plumbing without an agent run (Docker required, not part of `pnpm check`): diff --git a/apps/framework/scripts/run-vercel-evals.test.ts b/apps/framework/scripts/run-vercel-evals.test.ts index 1190490a..fe1b9806 100644 --- a/apps/framework/scripts/run-vercel-evals.test.ts +++ b/apps/framework/scripts/run-vercel-evals.test.ts @@ -11,8 +11,9 @@ import { } from 'node:fs'; import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; -import { describe, expect, it } from 'vitest'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { + agentEnvironment, cleanupSandbox, downloadResults, finalizeResult, @@ -25,6 +26,52 @@ import { expandJobs, } from './run-vercel-evals.js'; +const FORWARDED_ENV_NAMES = [ + 'ANTHROPIC_API_KEY', + 'OPENAI_API_KEY', + 'AI_GATEWAY_API_KEY', + 'SUPABASE_CLI_STABLE_VERSION', + 'SUPABASE_CLI_BETA_VERSION', +]; + +describe('agentEnvironment', () => { + const originalValues = new Map(); + + beforeEach(() => { + for (const name of FORWARDED_ENV_NAMES) { + originalValues.set(name, process.env[name]); + delete process.env[name]; + } + }); + + afterEach(() => { + for (const [name, value] of originalValues) { + if (value === undefined) delete process.env[name]; + else process.env[name] = value; + } + }); + + it('forwards every configured name when set', () => { + for (const name of FORWARDED_ENV_NAMES) { + process.env[name] = `${name}-value`; + } + + const lines = agentEnvironment().split('\n'); + for (const name of FORWARDED_ENV_NAMES) { + expect(lines).toContain(`${name}=${name}-value`); + } + }); + + it('omits the CLI channel pins when unset', () => { + process.env.ANTHROPIC_API_KEY = 'anthropic-value'; + + const env = agentEnvironment(); + expect(env).toBe('ANTHROPIC_API_KEY=anthropic-value'); + expect(env).not.toContain('SUPABASE_CLI_STABLE_VERSION'); + expect(env).not.toContain('SUPABASE_CLI_BETA_VERSION'); + }); +}); + describe('Vercel eval controller', () => { it('bounds concurrent work and lets independent failures settle', async () => { let active = 0; diff --git a/apps/framework/scripts/run-vercel-evals.ts b/apps/framework/scripts/run-vercel-evals.ts index 221cdcaa..efcbcc14 100644 --- a/apps/framework/scripts/run-vercel-evals.ts +++ b/apps/framework/scripts/run-vercel-evals.ts @@ -19,11 +19,16 @@ const ROOT = fileURLToPath(new URL('../../../', import.meta.url)); /** Base for sandbox URLs printed during runs */ const SANDBOX_DASHBOARD_URL = 'https://vercel.com/supabase/evals-runner/sandboxes'; -const AGENT_ENV_NAMES = [ +const FORWARDED_ENV_NAMES = [ 'ANTHROPIC_API_KEY', 'OPENAI_API_KEY', 'AI_GATEWAY_API_KEY', 'XAI_API_KEY', + // Pins the CLI channel versions the prepare job resolved for this run, so + // every sandbox job in the run scores against the same version instead of + // each independently re-resolving "latest" and drifting mid-run. + 'SUPABASE_CLI_STABLE_VERSION', + 'SUPABASE_CLI_BETA_VERSION', ]; /** * Slack for the non-agent work inside `pnpm eval` (supabase start, resets, @@ -622,10 +627,10 @@ function vercelCredentialsFromEnv(): { }; } -/** Serializes configured provider keys into the repo-root `.env` file. */ -function agentEnvironment(): string { +/** Serializes configured provider keys and CLI channel pins into the sandbox's `.env` file. */ +export function agentEnvironment(): string { const lines: string[] = []; - for (const name of AGENT_ENV_NAMES) { + for (const name of FORWARDED_ENV_NAMES) { const value = process.env[name]; if (value) lines.push(`${name}=${value}`); } diff --git a/packages/sandbox/package.json b/packages/sandbox/package.json index 41b587a1..9da33f75 100644 --- a/packages/sandbox/package.json +++ b/packages/sandbox/package.json @@ -11,7 +11,7 @@ }, "scripts": { "typecheck": "tsc --noEmit", - "test": "vitest run test/unit.test.ts", + "test": "vitest run test", "test:docker": "SANDBOX_DOCKER_TESTS=1 vitest run test/docker.test.ts" }, "dependencies": { diff --git a/packages/sandbox/src/cli-channel.ts b/packages/sandbox/src/cli-channel.ts new file mode 100644 index 00000000..768d913c --- /dev/null +++ b/packages/sandbox/src/cli-channel.ts @@ -0,0 +1,228 @@ +/** + * Resolves "latest stable" / "latest beta" Supabase CLI versions from npm's + * dist-tags for the `supabase` package — no auth, no rate-limit exposure + * (unlike the GitHub releases API), which matters because CI's sandbox only + * forwards the model API keys, never a GITHUB_TOKEN. + * + * npm's dist-tag can point at a version whose GitHub release is still a + * draft, so the resolved version's `.deb` release asset is HEAD-checked + * before being trusted; for the beta channel only, a missing asset walks + * back through older published `-beta.N` versions for one that downloads + * (see resolveFallbackBetaVersion). A stable dist-tag with a missing asset + * always throws instead — guessing at a "stable" release would defeat the + * point of the channel. + */ + +export type CliChannel = 'stable' | 'beta'; + +const NPM_DIST_TAGS_URL = + 'https://registry.npmjs.org/-/package/supabase/dist-tags'; + +// Abbreviated packument (versions + dist-tags only, no changelogs/READMEs) — +// this header is what makes npm serve the small form instead of the full one. +const NPM_PACKUMENT_URL = 'https://registry.npmjs.org/supabase'; +const NPM_INSTALL_V1_ACCEPT = 'application/vnd.npm.install-v1+json'; + +// npm's beta dist-tag can carry a prerelease suffix like -rc.1, not just -beta.N. +const VERSION_RE = /^\d+\.\d+\.\d+(-[0-9A-Za-z.]+)?$/; + +// Only the -beta.N shape is walkable for the "try an older published beta" +// fallback below; a -rc.N or other prerelease suffix has no defined ordering +// we can search. +const BETA_SUFFIX_RE = /^(\d+\.\d+\.\d+-beta\.)(\d+)$/; + +// The number of older published beta versions to probe for a downloadable +// asset before giving up — keeps a broken release from turning one nightly +// run into an unbounded chain of GitHub requests. +const MAX_BETA_FALLBACK_CANDIDATES = 5; + +const ENV_OVERRIDE: Record = { + stable: 'SUPABASE_CLI_STABLE_VERSION', + beta: 'SUPABASE_CLI_BETA_VERSION', +}; + +const DIST_TAG: Record = { + stable: 'latest', + beta: 'beta', +}; + +const versionCache = new Map>(); + +const CLI_CHANNELS = new Set(['stable', 'beta']); + +function isCliChannel(value: string): value is CliChannel { + return CLI_CHANNELS.has(value as CliChannel); +} + +/** Arch suffix the CLI's own release `.deb` filenames use. */ +export function hostDebArch(): 'amd64' | 'arm64' { + return process.arch === 'arm64' ? 'arm64' : 'amd64'; +} + +/** Mirrors installSupabaseCli's download URL template in packages/sandbox/src/supabase.ts. */ +export function cliDebUrl( + version: string, + arch: 'amd64' | 'arm64' = hostDebArch() +): string { + return `https://github.com/supabase/cli/releases/download/v${version}/supabase_${version}_linux_${arch}.deb`; +} + +/** HEAD-checks that a release's `.deb` asset is actually downloadable (not behind a draft release). */ +async function debAssetExists(version: string): Promise { + const response = await fetch(cliDebUrl(version), { + method: 'HEAD', + redirect: 'follow', + signal: AbortSignal.timeout(15_000), + }); + return response.ok; +} + +/** + * Resolve a channel to a concrete Supabase CLI version. Memoised per channel + * so a single nightly run only hits the registry once per channel; the cache + * entry is cleared on rejection so a later retry can hit the network again. + * Never falls back to the pinned SUPABASE_CLI_VERSION on failure — that would + * silently mislabel data — so callers must let the throw propagate. + */ +export async function resolveCliVersion(channel: CliChannel): Promise { + const cached = versionCache.get(channel); + if (cached) return cached; + + const promise = resolveCliVersionUncached(channel); + versionCache.set(channel, promise); + promise.catch(() => versionCache.delete(channel)); + return promise; +} + +/** + * Resolves `value` if it names a channel (`'stable'` | `'beta'`); an exact + * version (or `undefined`) passes through unchanged. Split out from + * localStackRuntime's startSession so the branching itself is unit-testable + * without booting a sandbox. + */ +export async function resolveCliVersionOption( + value: string | CliChannel | undefined +): Promise { + if (value === undefined) return undefined; + return isCliChannel(value) ? resolveCliVersion(value) : value; +} + +async function resolveCliVersionUncached(channel: CliChannel): Promise { + const envVar = ENV_OVERRIDE[channel]; + const override = process.env[envVar]?.trim().replace(/^v/, ''); + if (override) { + if (!VERSION_RE.test(override)) { + throw new Error( + `${envVar}=${JSON.stringify(override)} is not a valid Supabase CLI version` + ); + } + // An explicit pin is trusted as-is — no asset check, network or otherwise. + return override; + } + + const response = await fetch(NPM_DIST_TAGS_URL, { + signal: AbortSignal.timeout(15_000), + }); + if (!response.ok) { + throw new Error( + `failed to resolve the latest ${channel} Supabase CLI version: ` + + `GET ${NPM_DIST_TAGS_URL} -> ${response.status} ${response.statusText}` + ); + } + const tags = await response.json(); + const distTag = DIST_TAG[channel]; + const version = isRecord(tags) ? tags[distTag] : undefined; + if (typeof version !== 'string' || !VERSION_RE.test(version)) { + throw new Error( + `npm dist-tags for "supabase" did not have a valid "${distTag}" version ` + + `for the ${channel} channel: ${JSON.stringify(version)}` + ); + } + + if (await debAssetExists(version)) return version; + + // The dist-tag pointed at a version whose GitHub release has no + // downloadable .deb (e.g. still a draft) — this is exactly what broke the + // beta channel in CI run 35274970438. A stable release is never guessed at; + // only beta walks back to the newest published version that does have one. + if (channel === 'stable') { + throw new Error( + `npm's "latest" dist-tag for "supabase" points at ${version}, but its ` + + `release asset is missing: HEAD ${cliDebUrl(version)} was not ok` + ); + } + + return resolveFallbackBetaVersion(version); +} + +/** + * Walks back through published npm versions sharing the same `X.Y.Z-beta.` + * prefix as `unpublishedVersion`, newest first, and returns the first one + * whose release `.deb` asset actually downloads. + */ +async function resolveFallbackBetaVersion( + unpublishedVersion: string +): Promise { + const match = BETA_SUFFIX_RE.exec(unpublishedVersion); + if (!match) { + throw new Error( + `npm's "beta" dist-tag for "supabase" points at ${unpublishedVersion}, ` + + `whose release asset is missing (HEAD ${cliDebUrl(unpublishedVersion)} ` + + 'was not ok), and its version does not match the X.Y.Z-beta.N shape ' + + 'this fallback can walk back through' + ); + } + const [, prefix] = match; + + const response = await fetch(NPM_PACKUMENT_URL, { + headers: { Accept: NPM_INSTALL_V1_ACCEPT }, + signal: AbortSignal.timeout(15_000), + }); + if (!response.ok) { + throw new Error( + `failed to look up published beta versions for "supabase": ` + + `GET ${NPM_PACKUMENT_URL} -> ${response.status} ${response.statusText}` + ); + } + const packument = await response.json(); + const versions = isRecord(packument) ? packument.versions : undefined; + if (!isRecord(versions)) { + throw new Error( + `npm packument for "supabase" did not include a "versions" object` + ); + } + + const suffixRe = new RegExp(`^${escapeRegExp(prefix)}(\\d+)$`); + const candidates = Object.keys(versions) + .filter((candidate) => candidate !== unpublishedVersion) + .map((candidate) => { + const suffixMatch = suffixRe.exec(candidate); + return suffixMatch + ? { version: candidate, suffix: Number(suffixMatch[1]) } + : null; + }) + .filter((entry): entry is { version: string; suffix: number } => + Boolean(entry) + ) + .sort((a, b) => b.suffix - a.suffix) + .slice(0, MAX_BETA_FALLBACK_CANDIDATES) + .map((entry) => entry.version); + + for (const candidate of candidates) { + if (await debAssetExists(candidate)) return candidate; + } + + throw new Error( + `no published beta Supabase CLI release has a downloadable .deb asset; ` + + `checked ${[unpublishedVersion, ...candidates].join(', ')} via ` + + `${NPM_PACKUMENT_URL}` + ); +} + +function escapeRegExp(value: string): string { + return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null; +} diff --git a/packages/sandbox/src/index.ts b/packages/sandbox/src/index.ts index 793a5e5e..5cdc23ed 100644 --- a/packages/sandbox/src/index.ts +++ b/packages/sandbox/src/index.ts @@ -1,3 +1,5 @@ +export { resolveCliVersion, hostDebArch, cliDebUrl } from './cli-channel.js'; +export type { CliChannel } from './cli-channel.js'; export { DockerSandbox, dockerCli } from './docker-sandbox.js'; export type { DockerSandboxOptions, diff --git a/packages/sandbox/src/local-stack-runtime.ts b/packages/sandbox/src/local-stack-runtime.ts index 855a3be8..879f6720 100644 --- a/packages/sandbox/src/local-stack-runtime.ts +++ b/packages/sandbox/src/local-stack-runtime.ts @@ -15,6 +15,7 @@ import { DockerSandbox } from './docker-sandbox.js'; import { createAgentEnvironment } from './agent-environment.js'; import { ensureEdgeRuntime, teardownSupabaseProject } from './supabase.js'; import { buildSkillsPrompt } from './skills.js'; +import { resolveCliVersionOption, type CliChannel } from './cli-channel.js'; const DEFAULT_BASH_TIMEOUT_SEC = 240; const MAX_BASH_TIMEOUT_SEC = 600; @@ -36,8 +37,15 @@ const STACK_CONFIG_RETRY_MS = 2_000; * profile — out of scope for now. */ export interface LocalStackRuntimeOptions { - /** Supabase CLI version baked into the sandbox image (pinned default). */ - cliVersion?: string; + /** + * Supabase CLI version baked into the sandbox image (pinned default). + * Either an exact version (e.g. `2.109.1`) or a channel tag (`'stable'` | + * `'beta'`) that tracks npm's dist-tag for the `supabase` package — + * resolved to a concrete version once, at session start. An eval's own + * `cliVersion:` frontmatter pin always wins over this option, whether this + * is an exact version or a channel tag. + */ + cliVersion?: CliChannel | (string & {}); /** * Supabase MCP feature groups to expose to the agent when the eval links to * a hosted project (`hostedProject: true`). The MCP server runs host-side and @@ -86,7 +94,8 @@ export function localStackRuntime( skipCliInstall, }) { const env = await createAgentEnvironment({ - cliVersion: cliVersion ?? options.cliVersion, + cliVersion: + cliVersion ?? (await resolveCliVersionOption(options.cliVersion)), localDir, skills, mounts, diff --git a/packages/sandbox/test/cli-channel.test.ts b/packages/sandbox/test/cli-channel.test.ts new file mode 100644 index 00000000..209086e0 --- /dev/null +++ b/packages/sandbox/test/cli-channel.test.ts @@ -0,0 +1,352 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +const STABLE_ENV = 'SUPABASE_CLI_STABLE_VERSION'; +const BETA_ENV = 'SUPABASE_CLI_BETA_VERSION'; +const DIST_TAGS_URL = 'https://registry.npmjs.org/-/package/supabase/dist-tags'; +const PACKUMENT_URL = 'https://registry.npmjs.org/supabase'; +const INSTALL_V1_ACCEPT = 'application/vnd.npm.install-v1+json'; + +// Mirrors hostDebArch()'s mapping so this test's expected URLs match +// whatever host architecture actually runs it. +const HOST_ARCH = process.arch === 'arm64' ? 'arm64' : 'amd64'; + +function debUrl(version: string): string { + return `https://github.com/supabase/cli/releases/download/v${version}/supabase_${version}_linux_${HOST_ARCH}.deb`; +} + +function jsonResponse( + body: unknown, + init: { ok?: boolean; status?: number; statusText?: string } = {} +) { + return { + ok: init.ok ?? true, + status: init.status ?? 200, + statusText: init.statusText ?? 'OK', + json: async () => body, + }; +} + +function headResponse(ok: boolean) { + return { ok, status: ok ? 200 : 404, statusText: ok ? 'OK' : 'Not Found' }; +} + +/** + * Routes a single stubbed `fetch` by method + URL, mirroring the real + * mixture of GET (dist-tags, packument) and HEAD (asset check) calls + * resolveCliVersion makes. + */ +function routedFetchMock(routes: { + distTags?: unknown; + distTagsInit?: { ok?: boolean; status?: number; statusText?: string }; + assetOk?: (version: string) => boolean; + packument?: unknown; +}) { + return vi.fn(async (url: string, init?: RequestInit) => { + const method = init?.method ?? 'GET'; + if (method === 'HEAD') { + const version = url.match(/supabase_(.+)_linux_/)?.[1]; + return headResponse( + version ? (routes.assetOk?.(version) ?? false) : false + ); + } + if (url === DIST_TAGS_URL) { + return jsonResponse(routes.distTags, routes.distTagsInit); + } + if (url === PACKUMENT_URL) { + return jsonResponse(routes.packument); + } + throw new Error(`unexpected fetch: ${method} ${url}`); + }); +} + +beforeEach(() => { + vi.resetModules(); + delete process.env[STABLE_ENV]; + delete process.env[BETA_ENV]; +}); + +afterEach(() => { + vi.unstubAllGlobals(); + delete process.env[STABLE_ENV]; + delete process.env[BETA_ENV]; +}); + +describe('resolveCliVersion', () => { + it('resolves the stable channel from the "latest" dist-tag when its asset exists', async () => { + const fetchMock = routedFetchMock({ + distTags: { latest: '1.2.3', beta: '1.3.0-beta.1' }, + assetOk: () => true, + }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersion } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersion('stable')).resolves.toBe('1.2.3'); + + const headCalls = fetchMock.mock.calls.filter( + ([, init]) => init?.method === 'HEAD' + ); + expect(headCalls).toHaveLength(1); + expect(headCalls[0]?.[0]).toBe(debUrl('1.2.3')); + }); + + it('resolves the beta channel from the "beta" dist-tag when its asset exists', async () => { + const fetchMock = routedFetchMock({ + distTags: { latest: '1.2.3', beta: '1.3.0-beta.1' }, + assetOk: () => true, + }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersion } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersion('beta')).resolves.toBe('1.3.0-beta.1'); + }); + + it('prefers the env override over the network and strips a leading v', async () => { + process.env[STABLE_ENV] = 'v9.9.9'; + const fetchMock = vi.fn(); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersion } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersion('stable')).resolves.toBe('9.9.9'); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it('trims surrounding whitespace from the env override', async () => { + process.env[BETA_ENV] = ' 1.4.0-rc.2 '; + vi.stubGlobal('fetch', vi.fn()); + const { resolveCliVersion } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersion('beta')).resolves.toBe('1.4.0-rc.2'); + }); + + it('throws naming the env var when the override is not a valid version', async () => { + process.env[BETA_ENV] = 'not-a-version'; + vi.stubGlobal('fetch', vi.fn()); + const { resolveCliVersion } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersion('beta')).rejects.toThrow( + `${BETA_ENV}="not-a-version" is not a valid Supabase CLI version` + ); + }); + + it('throws with the HTTP status when the registry request fails', async () => { + const fetchMock = routedFetchMock({ + distTags: undefined, + distTagsInit: { + ok: false, + status: 500, + statusText: 'Internal Server Error', + }, + }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersion } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersion('stable')).rejects.toThrow( + `${DIST_TAGS_URL} -> 500 Internal Server Error` + ); + }); + + it('throws when the requested dist-tag is missing from the response', async () => { + const fetchMock = routedFetchMock({ distTags: { beta: '1.0.0-beta.1' } }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersion } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersion('stable')).rejects.toThrow( + /did not have a valid "latest" version for the stable channel/ + ); + }); + + it('throws when the dist-tag value is not a valid version string', async () => { + const fetchMock = routedFetchMock({ + distTags: { latest: 'not-a-version' }, + }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersion } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersion('stable')).rejects.toThrow( + /did not have a valid "latest" version for the stable channel/ + ); + }); + + it('memoises a successful resolution so a second call does not refetch', async () => { + const fetchMock = routedFetchMock({ + distTags: { latest: '1.2.3', beta: '1.3.0-beta.1' }, + assetOk: () => true, + }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersion } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersion('stable')).resolves.toBe('1.2.3'); + await expect(resolveCliVersion('stable')).resolves.toBe('1.2.3'); + + // One GET (dist-tags) + one HEAD (asset check) — the second call hits the cache. + expect(fetchMock).toHaveBeenCalledTimes(2); + }); + + it('clears the cache entry on rejection so a later call refetches', async () => { + let failNextDistTags = true; + const fetchMock = vi.fn(async (url: string, init?: RequestInit) => { + if (init?.method === 'HEAD') return headResponse(true); + if (url === DIST_TAGS_URL) { + if (failNextDistTags) { + failNextDistTags = false; + return jsonResponse(undefined, { + ok: false, + status: 500, + statusText: 'Internal Server Error', + }); + } + return jsonResponse({ latest: '1.2.3', beta: '1.3.0-beta.1' }); + } + throw new Error(`unexpected fetch: ${url}`); + }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersion } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersion('stable')).rejects.toThrow('500'); + await expect(resolveCliVersion('stable')).resolves.toBe('1.2.3'); + + expect(fetchMock).toHaveBeenCalledTimes(3); + }); + + it('does not let a beta rejection clear the stable cache entry', async () => { + const fetchMock = vi.fn(async (url: string, init?: RequestInit) => { + if (init?.method === 'HEAD') return headResponse(true); + if (url === DIST_TAGS_URL) { + return jsonResponse({ latest: '1.2.3', beta: 'not-a-version' }); + } + throw new Error(`unexpected fetch: ${url}`); + }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersion } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersion('stable')).resolves.toBe('1.2.3'); + await expect(resolveCliVersion('beta')).rejects.toThrow( + /did not have a valid "beta" version for the beta channel/ + ); + await expect(resolveCliVersion('stable')).resolves.toBe('1.2.3'); + + // Two dist-tags GETs (stable, beta) + one HEAD (stable's asset check + // only — beta never gets that far). + expect(fetchMock).toHaveBeenCalledTimes(3); + }); + + it('falls back to the newest published beta.N-1 when the dist-tag asset is a 404', async () => { + const fetchMock = routedFetchMock({ + distTags: { latest: '1.2.3', beta: '2.118.0-beta.52' }, + assetOk: (version) => version !== '2.118.0-beta.52', + packument: { + versions: { + '2.118.0-beta.52': {}, + '2.118.0-beta.51': {}, + '2.118.0-beta.50': {}, + '2.117.0': {}, + }, + }, + }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersion } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersion('beta')).resolves.toBe('2.118.0-beta.51'); + + const packumentCall = fetchMock.mock.calls.find( + ([url]) => url === PACKUMENT_URL + ); + expect(packumentCall?.[1]?.headers).toMatchObject({ + Accept: INSTALL_V1_ACCEPT, + }); + + const headUrls = fetchMock.mock.calls + .filter(([, init]) => init?.method === 'HEAD') + .map(([url]) => url); + expect(headUrls).toEqual([ + debUrl('2.118.0-beta.52'), + debUrl('2.118.0-beta.51'), + ]); + }); + + it('throws naming the unpublished versions when no published beta has a downloadable asset', async () => { + const fetchMock = routedFetchMock({ + distTags: { latest: '1.2.3', beta: '2.118.0-beta.52' }, + assetOk: () => false, + packument: { + versions: { + '2.118.0-beta.52': {}, + '2.118.0-beta.51': {}, + }, + }, + }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersion } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersion('beta')).rejects.toThrow( + /2\.118\.0-beta\.52.*2\.118\.0-beta\.51/ + ); + }); + + it('throws instead of guessing when the stable dist-tag asset is a 404', async () => { + const fetchMock = routedFetchMock({ + distTags: { latest: '1.2.3', beta: '1.3.0-beta.1' }, + assetOk: () => false, + }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersion } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersion('stable')).rejects.toThrow( + `HEAD ${debUrl('1.2.3')} was not ok` + ); + }); + + it('does not hit the network at all for an env override', async () => { + process.env[STABLE_ENV] = '9.9.9'; + process.env[BETA_ENV] = '9.9.9-beta.1'; + const fetchMock = vi.fn(); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersion } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersion('stable')).resolves.toBe('9.9.9'); + await expect(resolveCliVersion('beta')).resolves.toBe('9.9.9-beta.1'); + expect(fetchMock).not.toHaveBeenCalled(); + }); +}); + +describe('resolveCliVersionOption', () => { + it('passes an exact version through unchanged, without touching the network', async () => { + const fetchMock = vi.fn(); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersionOption } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersionOption('2.109.1')).resolves.toBe('2.109.1'); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it('passes undefined through unchanged, without touching the network', async () => { + const fetchMock = vi.fn(); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersionOption } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersionOption(undefined)).resolves.toBeUndefined(); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it('resolves a "stable" channel tag against npm', async () => { + const fetchMock = routedFetchMock({ + distTags: { latest: '1.2.3', beta: '1.3.0-beta.1' }, + assetOk: () => true, + }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersionOption } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersionOption('stable')).resolves.toBe('1.2.3'); + }); + + it('resolves a "beta" channel tag against npm', async () => { + const fetchMock = routedFetchMock({ + distTags: { latest: '1.2.3', beta: '1.3.0-beta.1' }, + assetOk: () => true, + }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersionOption } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersionOption('beta')).resolves.toBe('1.3.0-beta.1'); + }); +}); From f6753427d4428789ad012ee09f989c3a1c3a7f23 Mon Sep 17 00:00:00 2001 From: Colum Ferry Date: Tue, 22 Sep 2026 12:09:46 +0100 Subject: [PATCH 2/6] fix(sandbox): address PR #324 review findings for CLI channel resolution - Export FORWARDED_ENV_NAMES from run-vercel-evals.ts and import it in the test instead of a drifted local copy, so a leaked XAI_API_KEY can no longer break agentEnvironment()'s assertions. - Make cliDebUrl's arch param required and check both amd64 and arm64 release assets, since the resolver's host arch can differ from the sandbox container's; delete the now-unused hostDebArch. - Generalise the beta walk-back to every published X.Y.Z-beta.N version older than the unpublished one (not just the same minor), ordered with a numeric compareBetaVersionsDesc comparator instead of a string compare. - Treat a thrown error for one walk-back candidate as "unavailable" and keep probing the rest, instead of aborting the whole walk-back; the initial dist-tag asset check still fails loud. - Import isRecord from @supabase-evals/core/json instead of a local copy that dropped the array guard. - Guard the version cache's rejection cleanup with an identity check so a stale promise can't evict a newer cached one. - Drop cliDebUrl/hostDebArch from the sandbox barrel (no external consumers) and correct local-stack-runtime's doc comment: channel resolution is memoised per process, not per session. --- .../scripts/run-vercel-evals.test.ts | 9 +- apps/framework/scripts/run-vercel-evals.ts | 2 +- packages/core/package.json | 3 +- packages/core/src/json.ts | 7 +- packages/sandbox/src/cli-channel.ts | 147 +++++++---- packages/sandbox/src/index.ts | 2 +- packages/sandbox/src/local-stack-runtime.ts | 6 +- packages/sandbox/test/cli-channel.test.ts | 242 ++++++++++++++++-- 8 files changed, 336 insertions(+), 82 deletions(-) diff --git a/apps/framework/scripts/run-vercel-evals.test.ts b/apps/framework/scripts/run-vercel-evals.test.ts index fe1b9806..7da08d92 100644 --- a/apps/framework/scripts/run-vercel-evals.test.ts +++ b/apps/framework/scripts/run-vercel-evals.test.ts @@ -17,6 +17,7 @@ import { cleanupSandbox, downloadResults, finalizeResult, + FORWARDED_ENV_NAMES, isRetryableSandboxCreateError, isTerminalSandboxCreateError, packWorkspaceScript, @@ -26,14 +27,6 @@ import { expandJobs, } from './run-vercel-evals.js'; -const FORWARDED_ENV_NAMES = [ - 'ANTHROPIC_API_KEY', - 'OPENAI_API_KEY', - 'AI_GATEWAY_API_KEY', - 'SUPABASE_CLI_STABLE_VERSION', - 'SUPABASE_CLI_BETA_VERSION', -]; - describe('agentEnvironment', () => { const originalValues = new Map(); diff --git a/apps/framework/scripts/run-vercel-evals.ts b/apps/framework/scripts/run-vercel-evals.ts index efcbcc14..4ceccb2b 100644 --- a/apps/framework/scripts/run-vercel-evals.ts +++ b/apps/framework/scripts/run-vercel-evals.ts @@ -19,7 +19,7 @@ const ROOT = fileURLToPath(new URL('../../../', import.meta.url)); /** Base for sandbox URLs printed during runs */ const SANDBOX_DASHBOARD_URL = 'https://vercel.com/supabase/evals-runner/sandboxes'; -const FORWARDED_ENV_NAMES = [ +export const FORWARDED_ENV_NAMES = [ 'ANTHROPIC_API_KEY', 'OPENAI_API_KEY', 'AI_GATEWAY_API_KEY', diff --git a/packages/core/package.json b/packages/core/package.json index 51ed3f4d..6c41ea2e 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -8,7 +8,8 @@ "exports": { ".": "./src/index.ts", "./eval-metadata": "./src/eval-metadata.ts", - "./eval-markdown": "./src/eval-markdown.ts" + "./eval-markdown": "./src/eval-markdown.ts", + "./json": "./src/json.ts" }, "scripts": { "test": "vitest run" diff --git a/packages/core/src/json.ts b/packages/core/src/json.ts index b8da5d0d..8f7c0369 100644 --- a/packages/core/src/json.ts +++ b/packages/core/src/json.ts @@ -1,4 +1,9 @@ -/** Shared JSON helpers used across the core package. */ +/** + * Shared JSON helpers, imported both from within the core package and, via + * the `@supabase-evals/core/json` subpath, by other packages (e.g. + * `@supabase-evals/sandbox`'s CLI channel resolution) that want them without + * pulling in core's much larger `index.ts`. + */ export function isRecord(value: unknown): value is Record { return typeof value === 'object' && value !== null && !Array.isArray(value); diff --git a/packages/sandbox/src/cli-channel.ts b/packages/sandbox/src/cli-channel.ts index 768d913c..ffe87024 100644 --- a/packages/sandbox/src/cli-channel.ts +++ b/packages/sandbox/src/cli-channel.ts @@ -13,6 +13,8 @@ * point of the channel. */ +import { isRecord } from '@supabase-evals/core/json'; + export type CliChannel = 'stable' | 'beta'; const NPM_DIST_TAGS_URL = @@ -29,7 +31,12 @@ const VERSION_RE = /^\d+\.\d+\.\d+(-[0-9A-Za-z.]+)?$/; // Only the -beta.N shape is walkable for the "try an older published beta" // fallback below; a -rc.N or other prerelease suffix has no defined ordering // we can search. -const BETA_SUFFIX_RE = /^(\d+\.\d+\.\d+-beta\.)(\d+)$/; +const BETA_SUFFIX_RE = /^\d+\.\d+\.\d+-beta\.\d+$/; + +// Matches any published "X.Y.Z-beta.N" version (any major/minor/patch), +// captured for the numeric tuple comparison in compareBetaVersionsDesc — +// unlike BETA_SUFFIX_RE, it isn't anchored to one specific version's prefix. +const BETA_VERSION_RE = /^(\d+)\.(\d+)\.(\d+)-beta\.(\d+)$/; // The number of older published beta versions to probe for a downloadable // asset before giving up — keeps a broken release from turning one nightly @@ -54,22 +61,18 @@ function isCliChannel(value: string): value is CliChannel { return CLI_CHANNELS.has(value as CliChannel); } -/** Arch suffix the CLI's own release `.deb` filenames use. */ -export function hostDebArch(): 'amd64' | 'arm64' { - return process.arch === 'arm64' ? 'arm64' : 'amd64'; -} - /** Mirrors installSupabaseCli's download URL template in packages/sandbox/src/supabase.ts. */ -export function cliDebUrl( - version: string, - arch: 'amd64' | 'arm64' = hostDebArch() -): string { +export function cliDebUrl(version: string, arch: 'amd64' | 'arm64'): string { return `https://github.com/supabase/cli/releases/download/v${version}/supabase_${version}_linux_${arch}.deb`; } -/** HEAD-checks that a release's `.deb` asset is actually downloadable (not behind a draft release). */ -async function debAssetExists(version: string): Promise { - const response = await fetch(cliDebUrl(version), { +/** The GitHub release page for a version, named in errors instead of a single arch's asset URL. */ +function releaseTagUrl(version: string): string { + return `https://github.com/supabase/cli/releases/tag/v${version}`; +} + +async function debAssetHeadOk(url: string): Promise { + const response = await fetch(url, { method: 'HEAD', redirect: 'follow', signal: AbortSignal.timeout(15_000), @@ -77,10 +80,30 @@ async function debAssetExists(version: string): Promise { return response.ok; } +/** + * HEAD-checks that a release's `.deb` assets are actually downloadable (not + * behind a draft release) for both architectures the sandbox installs onto — + * the host running this resolver and the sandbox container it targets can + * differ (e.g. an Apple Silicon host building a linux/amd64 sandbox image), + * so checking only one arch could pass while the other 404s. This also + * catches a partially-uploaded release that a single-arch probe would miss. + * Two HEAD requests per candidate, capped at MAX_BETA_FALLBACK_CANDIDATES + * candidates in the walk-back below, is an acceptable request budget. + */ +async function debAssetExists(version: string): Promise { + const [amd64, arm64] = await Promise.all([ + debAssetHeadOk(cliDebUrl(version, 'amd64')), + debAssetHeadOk(cliDebUrl(version, 'arm64')), + ]); + return amd64 && arm64; +} + /** * Resolve a channel to a concrete Supabase CLI version. Memoised per channel * so a single nightly run only hits the registry once per channel; the cache - * entry is cleared on rejection so a later retry can hit the network again. + * entry is cleared on rejection so a later retry can hit the network again — + * guarded by an identity check so a stale rejection can never evict a + * different (newer) promise that has since taken its place in the cache. * Never falls back to the pinned SUPABASE_CLI_VERSION on failure — that would * silently mislabel data — so callers must let the throw propagate. */ @@ -90,7 +113,9 @@ export async function resolveCliVersion(channel: CliChannel): Promise { const promise = resolveCliVersionUncached(channel); versionCache.set(channel, promise); - promise.catch(() => versionCache.delete(channel)); + promise.catch(() => { + if (versionCache.get(channel) === promise) versionCache.delete(channel); + }); return promise; } @@ -139,6 +164,9 @@ async function resolveCliVersionUncached(channel: CliChannel): Promise { ); } + // Deliberately not caught: a transient failure here (DNS blip, timeout) + // must throw loud rather than silently walk back to an older beta, or + // (for stable) never walk back at all. if (await debAssetExists(version)) return version; // The dist-tag pointed at a version whose GitHub release has no @@ -148,7 +176,8 @@ async function resolveCliVersionUncached(channel: CliChannel): Promise { if (channel === 'stable') { throw new Error( `npm's "latest" dist-tag for "supabase" points at ${version}, but its ` + - `release asset is missing: HEAD ${cliDebUrl(version)} was not ok` + `release asset is missing (checked amd64 and arm64 .deb assets at ` + + `${releaseTagUrl(version)})` ); } @@ -156,23 +185,30 @@ async function resolveCliVersionUncached(channel: CliChannel): Promise { } /** - * Walks back through published npm versions sharing the same `X.Y.Z-beta.` - * prefix as `unpublishedVersion`, newest first, and returns the first one - * whose release `.deb` asset actually downloads. + * Walks back through every published npm version matching the `X.Y.Z-beta.N` + * shape that is strictly older than `unpublishedVersion` — not just versions + * sharing its exact minor — newest first, and returns the first one whose + * release `.deb` assets actually download. This lets a brand-new minor's + * first beta (e.g. `2.119.0-beta.1`, still a draft) fall back across the + * minor boundary to the previous minor's newest published beta (e.g. + * `2.118.0-beta.60`). A candidate newer than `unpublishedVersion` is never + * considered — if npm's dist-tag skipped it, it's likelier to be a draft too. + * + * A transient error probing one candidate (network blip, timeout) is logged + * and skipped rather than aborting the whole walk-back, so one bad candidate + * doesn't waste the rest of the probe budget. */ async function resolveFallbackBetaVersion( unpublishedVersion: string ): Promise { - const match = BETA_SUFFIX_RE.exec(unpublishedVersion); - if (!match) { + if (!BETA_SUFFIX_RE.test(unpublishedVersion)) { throw new Error( `npm's "beta" dist-tag for "supabase" points at ${unpublishedVersion}, ` + - `whose release asset is missing (HEAD ${cliDebUrl(unpublishedVersion)} ` + - 'was not ok), and its version does not match the X.Y.Z-beta.N shape ' + - 'this fallback can walk back through' + `whose release asset is missing (checked amd64 and arm64 .deb assets ` + + `at ${releaseTagUrl(unpublishedVersion)}), and its version does not ` + + 'match the X.Y.Z-beta.N shape this fallback can walk back through' ); } - const [, prefix] = match; const response = await fetch(NPM_PACKUMENT_URL, { headers: { Accept: NPM_INSTALL_V1_ACCEPT }, @@ -192,24 +228,23 @@ async function resolveFallbackBetaVersion( ); } - const suffixRe = new RegExp(`^${escapeRegExp(prefix)}(\\d+)$`); const candidates = Object.keys(versions) - .filter((candidate) => candidate !== unpublishedVersion) - .map((candidate) => { - const suffixMatch = suffixRe.exec(candidate); - return suffixMatch - ? { version: candidate, suffix: Number(suffixMatch[1]) } - : null; - }) - .filter((entry): entry is { version: string; suffix: number } => - Boolean(entry) + .filter((candidate) => BETA_VERSION_RE.test(candidate)) + .filter( + (candidate) => compareBetaVersionsDesc(candidate, unpublishedVersion) > 0 ) - .sort((a, b) => b.suffix - a.suffix) - .slice(0, MAX_BETA_FALLBACK_CANDIDATES) - .map((entry) => entry.version); + .sort(compareBetaVersionsDesc) + .slice(0, MAX_BETA_FALLBACK_CANDIDATES); for (const candidate of candidates) { - if (await debAssetExists(candidate)) return candidate; + try { + if (await debAssetExists(candidate)) return candidate; + } catch (error) { + console.warn( + `[cli-channel] beta fallback candidate ${candidate} could not be checked, skipping: ` + + (error instanceof Error ? error.message : String(error)) + ); + } } throw new Error( @@ -219,10 +254,36 @@ async function resolveFallbackBetaVersion( ); } -function escapeRegExp(value: string): string { - return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +function parseBetaVersion( + version: string +): [major: number, minor: number, patch: number, beta: number] | undefined { + const match = BETA_VERSION_RE.exec(version); + if (!match) return undefined; + return [ + Number(match[1]), + Number(match[2]), + Number(match[3]), + Number(match[4]), + ]; } -function isRecord(value: unknown): value is Record { - return typeof value === 'object' && value !== null; +/** + * Orders two "X.Y.Z-beta.N" version strings newest-first — usable directly as + * an `Array#sort` comparator for descending order. Compares each component + * numerically so e.g. `2.118.0-beta.10` correctly sorts ahead of + * `2.118.0-beta.9`; a plain string compare would invert that. Throws if + * either string isn't a parseable `X.Y.Z-beta.N` version. + */ +export function compareBetaVersionsDesc(a: string, b: string): number { + const tupleA = parseBetaVersion(a); + const tupleB = parseBetaVersion(b); + if (!tupleA || !tupleB) { + throw new Error( + `compareBetaVersionsDesc expected "X.Y.Z-beta.N" versions, got ${JSON.stringify(a)} and ${JSON.stringify(b)}` + ); + } + for (let index = 0; index < tupleA.length; index += 1) { + if (tupleA[index] !== tupleB[index]) return tupleB[index] - tupleA[index]; + } + return 0; } diff --git a/packages/sandbox/src/index.ts b/packages/sandbox/src/index.ts index 5cdc23ed..52d68e09 100644 --- a/packages/sandbox/src/index.ts +++ b/packages/sandbox/src/index.ts @@ -1,4 +1,4 @@ -export { resolveCliVersion, hostDebArch, cliDebUrl } from './cli-channel.js'; +export { resolveCliVersion } from './cli-channel.js'; export type { CliChannel } from './cli-channel.js'; export { DockerSandbox, dockerCli } from './docker-sandbox.js'; export type { diff --git a/packages/sandbox/src/local-stack-runtime.ts b/packages/sandbox/src/local-stack-runtime.ts index 879f6720..392d4dad 100644 --- a/packages/sandbox/src/local-stack-runtime.ts +++ b/packages/sandbox/src/local-stack-runtime.ts @@ -40,8 +40,10 @@ export interface LocalStackRuntimeOptions { /** * Supabase CLI version baked into the sandbox image (pinned default). * Either an exact version (e.g. `2.109.1`) or a channel tag (`'stable'` | - * `'beta'`) that tracks npm's dist-tag for the `supabase` package — - * resolved to a concrete version once, at session start. An eval's own + * `'beta'`) that tracks npm's dist-tag for the `supabase` package — a + * channel tag's resolution is memoised for the lifetime of the process + * (see resolveCliVersion), not per session, so a later session in the same + * run will not pick up a newly published version. An eval's own * `cliVersion:` frontmatter pin always wins over this option, whether this * is an exact version or a channel tag. */ diff --git a/packages/sandbox/test/cli-channel.test.ts b/packages/sandbox/test/cli-channel.test.ts index 209086e0..8bc1353a 100644 --- a/packages/sandbox/test/cli-channel.test.ts +++ b/packages/sandbox/test/cli-channel.test.ts @@ -1,4 +1,5 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { cliDebUrl } from '../src/cli-channel.js'; const STABLE_ENV = 'SUPABASE_CLI_STABLE_VERSION'; const BETA_ENV = 'SUPABASE_CLI_BETA_VERSION'; @@ -6,14 +7,6 @@ const DIST_TAGS_URL = 'https://registry.npmjs.org/-/package/supabase/dist-tags'; const PACKUMENT_URL = 'https://registry.npmjs.org/supabase'; const INSTALL_V1_ACCEPT = 'application/vnd.npm.install-v1+json'; -// Mirrors hostDebArch()'s mapping so this test's expected URLs match -// whatever host architecture actually runs it. -const HOST_ARCH = process.arch === 'arm64' ? 'arm64' : 'amd64'; - -function debUrl(version: string): string { - return `https://github.com/supabase/cli/releases/download/v${version}/supabase_${version}_linux_${HOST_ARCH}.deb`; -} - function jsonResponse( body: unknown, init: { ok?: boolean; status?: number; statusText?: string } = {} @@ -32,21 +25,24 @@ function headResponse(ok: boolean) { /** * Routes a single stubbed `fetch` by method + URL, mirroring the real - * mixture of GET (dist-tags, packument) and HEAD (asset check) calls - * resolveCliVersion makes. + * mixture of GET (dist-tags, packument) and HEAD (amd64 + arm64 asset check) + * calls resolveCliVersion makes. */ function routedFetchMock(routes: { distTags?: unknown; distTagsInit?: { ok?: boolean; status?: number; statusText?: string }; - assetOk?: (version: string) => boolean; + assetOk?: (version: string, arch: 'amd64' | 'arm64') => boolean; packument?: unknown; }) { return vi.fn(async (url: string, init?: RequestInit) => { const method = init?.method ?? 'GET'; if (method === 'HEAD') { - const version = url.match(/supabase_(.+)_linux_/)?.[1]; + const match = url.match(/supabase_(.+)_linux_(amd64|arm64)\.deb$/); + const [, version, arch] = match ?? []; return headResponse( - version ? (routes.assetOk?.(version) ?? false) : false + version && arch + ? (routes.assetOk?.(version, arch as 'amd64' | 'arm64') ?? false) + : false ); } if (url === DIST_TAGS_URL) { @@ -85,8 +81,10 @@ describe('resolveCliVersion', () => { const headCalls = fetchMock.mock.calls.filter( ([, init]) => init?.method === 'HEAD' ); - expect(headCalls).toHaveLength(1); - expect(headCalls[0]?.[0]).toBe(debUrl('1.2.3')); + expect(headCalls.map(([url]) => url)).toEqual([ + cliDebUrl('1.2.3', 'amd64'), + cliDebUrl('1.2.3', 'arm64'), + ]); }); it('resolves the beta channel from the "beta" dist-tag when its asset exists', async () => { @@ -167,6 +165,16 @@ describe('resolveCliVersion', () => { ); }); + it('treats an array dist-tags response as invalid rather than a record', async () => { + const fetchMock = routedFetchMock({ distTags: ['not', 'a', 'record'] }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersion } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersion('stable')).rejects.toThrow( + /did not have a valid "latest" version for the stable channel/ + ); + }); + it('memoises a successful resolution so a second call does not refetch', async () => { const fetchMock = routedFetchMock({ distTags: { latest: '1.2.3', beta: '1.3.0-beta.1' }, @@ -178,11 +186,12 @@ describe('resolveCliVersion', () => { await expect(resolveCliVersion('stable')).resolves.toBe('1.2.3'); await expect(resolveCliVersion('stable')).resolves.toBe('1.2.3'); - // One GET (dist-tags) + one HEAD (asset check) — the second call hits the cache. - expect(fetchMock).toHaveBeenCalledTimes(2); + // One GET (dist-tags) + two HEAD (amd64+arm64 asset check) — the second + // call hits the cache. + expect(fetchMock).toHaveBeenCalledTimes(3); }); - it('clears the cache entry on rejection so a later call refetches', async () => { + it('clears the cache entry on rejection so a later call refetches, and keeps the retry memoised', async () => { let failNextDistTags = true; const fetchMock = vi.fn(async (url: string, init?: RequestInit) => { if (init?.method === 'HEAD') return headResponse(true); @@ -204,8 +213,11 @@ describe('resolveCliVersion', () => { await expect(resolveCliVersion('stable')).rejects.toThrow('500'); await expect(resolveCliVersion('stable')).resolves.toBe('1.2.3'); + // A third call must still hit the cache populated by the successful + // retry — the rejected first promise's cleanup must not evict it. + await expect(resolveCliVersion('stable')).resolves.toBe('1.2.3'); - expect(fetchMock).toHaveBeenCalledTimes(3); + expect(fetchMock).toHaveBeenCalledTimes(4); }); it('does not let a beta rejection clear the stable cache entry', async () => { @@ -225,9 +237,9 @@ describe('resolveCliVersion', () => { ); await expect(resolveCliVersion('stable')).resolves.toBe('1.2.3'); - // Two dist-tags GETs (stable, beta) + one HEAD (stable's asset check - // only — beta never gets that far). - expect(fetchMock).toHaveBeenCalledTimes(3); + // Two dist-tags GETs (stable, beta) + two HEAD (stable's amd64+arm64 + // asset check only — beta never gets that far). + expect(fetchMock).toHaveBeenCalledTimes(4); }); it('falls back to the newest published beta.N-1 when the dist-tag asset is a 404', async () => { @@ -259,8 +271,152 @@ describe('resolveCliVersion', () => { .filter(([, init]) => init?.method === 'HEAD') .map(([url]) => url); expect(headUrls).toEqual([ - debUrl('2.118.0-beta.52'), - debUrl('2.118.0-beta.51'), + cliDebUrl('2.118.0-beta.52', 'amd64'), + cliDebUrl('2.118.0-beta.52', 'arm64'), + cliDebUrl('2.118.0-beta.51', 'amd64'), + cliDebUrl('2.118.0-beta.51', 'arm64'), + ]); + }); + + it("falls back across a minor version boundary to the previous minor's newest beta", async () => { + const fetchMock = routedFetchMock({ + distTags: { latest: '2.118.0', beta: '2.119.0-beta.1' }, + assetOk: (version) => version === '2.118.0-beta.60', + packument: { + versions: { + '2.119.0-beta.1': {}, + '2.118.0-beta.60': {}, + '2.118.0-beta.59': {}, + '2.117.0': {}, + }, + }, + }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersion } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersion('beta')).resolves.toBe('2.118.0-beta.60'); + }); + + it('orders beta.10 ahead of beta.9 during the walk-back (not a string compare)', async () => { + const fetchMock = routedFetchMock({ + distTags: { latest: '1.2.3', beta: '2.118.0-beta.11' }, + // Only the unpublished dist-tag version (11) 404s; both walk-back + // candidates would succeed, so probe order is what decides the result. + assetOk: (version) => version !== '2.118.0-beta.11', + packument: { + versions: { + '2.118.0-beta.11': {}, + '2.118.0-beta.10': {}, + '2.118.0-beta.9': {}, + }, + }, + }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersion } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersion('beta')).resolves.toBe('2.118.0-beta.10'); + + const headUrls = fetchMock.mock.calls + .filter(([, init]) => init?.method === 'HEAD') + .map(([url]) => url); + // A naive string compare would sort "beta.9" ahead of "beta.10" (since + // '9' > '1' character-wise), probing 9 before 10. + expect(headUrls).toEqual([ + cliDebUrl('2.118.0-beta.11', 'amd64'), + cliDebUrl('2.118.0-beta.11', 'arm64'), + cliDebUrl('2.118.0-beta.10', 'amd64'), + cliDebUrl('2.118.0-beta.10', 'arm64'), + ]); + }); + + it('never selects a beta candidate newer than the unpublished dist-tag version', async () => { + const fetchMock = routedFetchMock({ + distTags: { latest: '1.2.3', beta: '2.118.0-beta.50' }, + // Every candidate downloads except the unpublished one — if the newer + // candidate (51) were ever probed, this would resolve to beta.51 + // instead of walking further back to beta.49. + assetOk: (version) => version !== '2.118.0-beta.50', + packument: { + versions: { + '2.118.0-beta.51': {}, + '2.118.0-beta.50': {}, + '2.118.0-beta.49': {}, + }, + }, + }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersion } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersion('beta')).resolves.toBe('2.118.0-beta.49'); + }); + + it('treats a walk-back candidate whose asset check throws as unavailable and continues to the next', async () => { + const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}); + const fetchMock = vi.fn(async (url: string, init?: RequestInit) => { + if (init?.method === 'HEAD') { + if (url.includes('2.118.0-beta.52')) return headResponse(false); + if (url.includes('2.118.0-beta.51')) throw new Error('network blip'); + if (url.includes('2.118.0-beta.50')) return headResponse(true); + return headResponse(false); + } + if (url === DIST_TAGS_URL) { + return jsonResponse({ latest: '1.2.3', beta: '2.118.0-beta.52' }); + } + if (url === PACKUMENT_URL) { + return jsonResponse({ + versions: { + '2.118.0-beta.52': {}, + '2.118.0-beta.51': {}, + '2.118.0-beta.50': {}, + }, + }); + } + throw new Error(`unexpected fetch: ${url}`); + }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersion } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersion('beta')).resolves.toBe('2.118.0-beta.50'); + expect(warnSpy).toHaveBeenCalledWith( + expect.stringContaining('2.118.0-beta.51') + ); + expect(warnSpy).toHaveBeenCalledWith( + expect.stringContaining('network blip') + ); + + warnSpy.mockRestore(); + }); + + it('caps the beta walk-back at MAX_BETA_FALLBACK_CANDIDATES, probing newest-first', async () => { + const versions: Record = {}; + for (let n = 50; n <= 59; n += 1) versions[`2.118.0-beta.${n}`] = {}; + const fetchMock = routedFetchMock({ + distTags: { latest: '1.2.3', beta: '2.118.0-beta.60' }, + assetOk: () => false, + packument: { versions }, + }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersion } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersion('beta')).rejects.toThrow(); + + const probedVersions = [ + ...new Set( + fetchMock.mock.calls + .filter(([, init]) => init?.method === 'HEAD') + .map(([url]) => url.match(/supabase_(.+)_linux_/)?.[1]) + ), + ]; + // The unpublished dist-tag version itself is checked first (outside the + // walk-back budget), followed by exactly MAX_BETA_FALLBACK_CANDIDATES (5) + // older published betas, newest-first. + expect(probedVersions).toEqual([ + '2.118.0-beta.60', + '2.118.0-beta.59', + '2.118.0-beta.58', + '2.118.0-beta.57', + '2.118.0-beta.56', + '2.118.0-beta.55', ]); }); @@ -292,7 +448,8 @@ describe('resolveCliVersion', () => { const { resolveCliVersion } = await import('../src/cli-channel.js'); await expect(resolveCliVersion('stable')).rejects.toThrow( - `HEAD ${debUrl('1.2.3')} was not ok` + 'checked amd64 and arm64 .deb assets at ' + + 'https://github.com/supabase/cli/releases/tag/v1.2.3' ); }); @@ -350,3 +507,38 @@ describe('resolveCliVersionOption', () => { await expect(resolveCliVersionOption('beta')).resolves.toBe('1.3.0-beta.1'); }); }); + +describe('compareBetaVersionsDesc', () => { + it('sorts newer beta numbers before older ones within the same minor', async () => { + const { compareBetaVersionsDesc } = await import('../src/cli-channel.js'); + + expect( + compareBetaVersionsDesc('2.118.0-beta.10', '2.118.0-beta.9') + ).toBeLessThan(0); + expect( + compareBetaVersionsDesc('2.118.0-beta.9', '2.118.0-beta.10') + ).toBeGreaterThan(0); + }); + + it('sorts a newer minor before an older minor regardless of beta number', async () => { + const { compareBetaVersionsDesc } = await import('../src/cli-channel.js'); + + expect( + compareBetaVersionsDesc('2.119.0-beta.1', '2.118.0-beta.60') + ).toBeLessThan(0); + }); + + it('treats equal versions as equal', async () => { + const { compareBetaVersionsDesc } = await import('../src/cli-channel.js'); + + expect(compareBetaVersionsDesc('2.118.0-beta.1', '2.118.0-beta.1')).toBe(0); + }); + + it('throws for a non "X.Y.Z-beta.N" version', async () => { + const { compareBetaVersionsDesc } = await import('../src/cli-channel.js'); + + expect(() => + compareBetaVersionsDesc('2.118.0-rc.1', '2.118.0-beta.1') + ).toThrow(); + }); +}); From 1518af50ea3246366c151385bfa2f4227de878ff Mon Sep 17 00:00:00 2001 From: Colum Ferry Date: Tue, 22 Sep 2026 12:30:36 +0100 Subject: [PATCH 3/6] chore(sandbox): trim narration and provenance comments in CLI channel resolution Strips agent-added comment cruft (module docblock essay, CI-run-ID call-out, code-shape narration) back to the invariants and external quirks the code can't express on its own; no behaviour change. --- apps/framework/scripts/run-vercel-evals.ts | 4 +- packages/sandbox/src/cli-channel.ts | 99 ++++++--------------- packages/sandbox/src/local-stack-runtime.ts | 12 +-- 3 files changed, 32 insertions(+), 83 deletions(-) diff --git a/apps/framework/scripts/run-vercel-evals.ts b/apps/framework/scripts/run-vercel-evals.ts index 4ceccb2b..ee231c2b 100644 --- a/apps/framework/scripts/run-vercel-evals.ts +++ b/apps/framework/scripts/run-vercel-evals.ts @@ -24,9 +24,7 @@ export const FORWARDED_ENV_NAMES = [ 'OPENAI_API_KEY', 'AI_GATEWAY_API_KEY', 'XAI_API_KEY', - // Pins the CLI channel versions the prepare job resolved for this run, so - // every sandbox job in the run scores against the same version instead of - // each independently re-resolving "latest" and drifting mid-run. + // Pins the CLI channel version resolved for this run across sandbox jobs. 'SUPABASE_CLI_STABLE_VERSION', 'SUPABASE_CLI_BETA_VERSION', ]; diff --git a/packages/sandbox/src/cli-channel.ts b/packages/sandbox/src/cli-channel.ts index ffe87024..4990a4e0 100644 --- a/packages/sandbox/src/cli-channel.ts +++ b/packages/sandbox/src/cli-channel.ts @@ -1,16 +1,7 @@ /** - * Resolves "latest stable" / "latest beta" Supabase CLI versions from npm's - * dist-tags for the `supabase` package — no auth, no rate-limit exposure - * (unlike the GitHub releases API), which matters because CI's sandbox only - * forwards the model API keys, never a GITHUB_TOKEN. - * - * npm's dist-tag can point at a version whose GitHub release is still a - * draft, so the resolved version's `.deb` release asset is HEAD-checked - * before being trusted; for the beta channel only, a missing asset walks - * back through older published `-beta.N` versions for one that downloads - * (see resolveFallbackBetaVersion). A stable dist-tag with a missing asset - * always throws instead — guessing at a "stable" release would defeat the - * point of the channel. + * Resolves "latest stable"/"latest beta" Supabase CLI versions from npm's + * dist-tags. npm can point at a still-draft release with no downloadable + * asset, so the resolved version's `.deb` is verified first. */ import { isRecord } from '@supabase-evals/core/json'; @@ -20,27 +11,20 @@ export type CliChannel = 'stable' | 'beta'; const NPM_DIST_TAGS_URL = 'https://registry.npmjs.org/-/package/supabase/dist-tags'; -// Abbreviated packument (versions + dist-tags only, no changelogs/READMEs) — -// this header is what makes npm serve the small form instead of the full one. +// Makes npm return the abbreviated packument instead of the full one. const NPM_PACKUMENT_URL = 'https://registry.npmjs.org/supabase'; const NPM_INSTALL_V1_ACCEPT = 'application/vnd.npm.install-v1+json'; // npm's beta dist-tag can carry a prerelease suffix like -rc.1, not just -beta.N. const VERSION_RE = /^\d+\.\d+\.\d+(-[0-9A-Za-z.]+)?$/; -// Only the -beta.N shape is walkable for the "try an older published beta" -// fallback below; a -rc.N or other prerelease suffix has no defined ordering -// we can search. +// Only the -beta.N shape has a defined ordering the walk-back below can search. const BETA_SUFFIX_RE = /^\d+\.\d+\.\d+-beta\.\d+$/; -// Matches any published "X.Y.Z-beta.N" version (any major/minor/patch), -// captured for the numeric tuple comparison in compareBetaVersionsDesc — -// unlike BETA_SUFFIX_RE, it isn't anchored to one specific version's prefix. +// Feeds compareBetaVersionsDesc's numeric comparison; matches any version, unlike BETA_SUFFIX_RE. const BETA_VERSION_RE = /^(\d+)\.(\d+)\.(\d+)-beta\.(\d+)$/; -// The number of older published beta versions to probe for a downloadable -// asset before giving up — keeps a broken release from turning one nightly -// run into an unbounded chain of GitHub requests. +// Caps the walk-back so a broken release can't chain into unbounded GitHub requests. const MAX_BETA_FALLBACK_CANDIDATES = 5; const ENV_OVERRIDE: Record = { @@ -61,12 +45,12 @@ function isCliChannel(value: string): value is CliChannel { return CLI_CHANNELS.has(value as CliChannel); } -/** Mirrors installSupabaseCli's download URL template in packages/sandbox/src/supabase.ts. */ +/** Must match the download URL installSupabaseCli() uses in supabase.ts. */ export function cliDebUrl(version: string, arch: 'amd64' | 'arm64'): string { return `https://github.com/supabase/cli/releases/download/v${version}/supabase_${version}_linux_${arch}.deb`; } -/** The GitHub release page for a version, named in errors instead of a single arch's asset URL. */ +/** GitHub release page for a version, used in error messages. */ function releaseTagUrl(version: string): string { return `https://github.com/supabase/cli/releases/tag/v${version}`; } @@ -80,16 +64,7 @@ async function debAssetHeadOk(url: string): Promise { return response.ok; } -/** - * HEAD-checks that a release's `.deb` assets are actually downloadable (not - * behind a draft release) for both architectures the sandbox installs onto — - * the host running this resolver and the sandbox container it targets can - * differ (e.g. an Apple Silicon host building a linux/amd64 sandbox image), - * so checking only one arch could pass while the other 404s. This also - * catches a partially-uploaded release that a single-arch probe would miss. - * Two HEAD requests per candidate, capped at MAX_BETA_FALLBACK_CANDIDATES - * candidates in the walk-back below, is an acceptable request budget. - */ +/** HEAD-checks both the amd64 and arm64 `.deb` assets, since the resolver's host architecture need not match the sandbox's. */ async function debAssetExists(version: string): Promise { const [amd64, arm64] = await Promise.all([ debAssetHeadOk(cliDebUrl(version, 'amd64')), @@ -99,13 +74,10 @@ async function debAssetExists(version: string): Promise { } /** - * Resolve a channel to a concrete Supabase CLI version. Memoised per channel - * so a single nightly run only hits the registry once per channel; the cache - * entry is cleared on rejection so a later retry can hit the network again — - * guarded by an identity check so a stale rejection can never evict a - * different (newer) promise that has since taken its place in the cache. - * Never falls back to the pinned SUPABASE_CLI_VERSION on failure — that would - * silently mislabel data — so callers must let the throw propagate. + * Resolves a channel to a concrete CLI version, memoised per channel for the + * process lifetime. A rejection clears the cache entry so the next call + * retries; never falls back to the pinned SUPABASE_CLI_VERSION, since that + * would silently mislabel data. */ export async function resolveCliVersion(channel: CliChannel): Promise { const cached = versionCache.get(channel); @@ -119,12 +91,7 @@ export async function resolveCliVersion(channel: CliChannel): Promise { return promise; } -/** - * Resolves `value` if it names a channel (`'stable'` | `'beta'`); an exact - * version (or `undefined`) passes through unchanged. Split out from - * localStackRuntime's startSession so the branching itself is unit-testable - * without booting a sandbox. - */ +/** Resolves `value`: a channel tag (`'stable'` | `'beta'`) resolves against npm; an exact version or `undefined` passes through unchanged. */ export async function resolveCliVersionOption( value: string | CliChannel | undefined ): Promise { @@ -164,15 +131,13 @@ async function resolveCliVersionUncached(channel: CliChannel): Promise { ); } - // Deliberately not caught: a transient failure here (DNS blip, timeout) - // must throw loud rather than silently walk back to an older beta, or - // (for stable) never walk back at all. + // A transient failure here throws rather than silently walking back to an + // older beta (or never walking back, for stable). if (await debAssetExists(version)) return version; - // The dist-tag pointed at a version whose GitHub release has no - // downloadable .deb (e.g. still a draft) — this is exactly what broke the - // beta channel in CI run 35274970438. A stable release is never guessed at; - // only beta walks back to the newest published version that does have one. + // npm's dist-tag can point at a version whose GitHub release is still a + // draft with no downloadable .deb. Only beta walks back to an older + // published version; stable is never guessed at. if (channel === 'stable') { throw new Error( `npm's "latest" dist-tag for "supabase" points at ${version}, but its ` + @@ -185,18 +150,10 @@ async function resolveCliVersionUncached(channel: CliChannel): Promise { } /** - * Walks back through every published npm version matching the `X.Y.Z-beta.N` - * shape that is strictly older than `unpublishedVersion` — not just versions - * sharing its exact minor — newest first, and returns the first one whose - * release `.deb` assets actually download. This lets a brand-new minor's - * first beta (e.g. `2.119.0-beta.1`, still a draft) fall back across the - * minor boundary to the previous minor's newest published beta (e.g. - * `2.118.0-beta.60`). A candidate newer than `unpublishedVersion` is never - * considered — if npm's dist-tag skipped it, it's likelier to be a draft too. - * - * A transient error probing one candidate (network blip, timeout) is logged - * and skipped rather than aborting the whole walk-back, so one bad candidate - * doesn't waste the rest of the probe budget. + * Walks back through published `X.Y.Z-beta.N` versions strictly older than + * `unpublishedVersion`, newest first, for one with a downloadable `.deb`. + * Newer candidates are skipped too (likelier to be drafts); a transient + * probe error is logged and skipped rather than aborting the walk-back. */ async function resolveFallbackBetaVersion( unpublishedVersion: string @@ -268,11 +225,9 @@ function parseBetaVersion( } /** - * Orders two "X.Y.Z-beta.N" version strings newest-first — usable directly as - * an `Array#sort` comparator for descending order. Compares each component - * numerically so e.g. `2.118.0-beta.10` correctly sorts ahead of - * `2.118.0-beta.9`; a plain string compare would invert that. Throws if - * either string isn't a parseable `X.Y.Z-beta.N` version. + * Orders two `X.Y.Z-beta.N` versions newest-first, usable as an + * `Array#sort` comparator; compares components numerically so `beta.10` + * sorts ahead of `beta.9`. Throws if either string isn't parseable. */ export function compareBetaVersionsDesc(a: string, b: string): number { const tupleA = parseBetaVersion(a); diff --git a/packages/sandbox/src/local-stack-runtime.ts b/packages/sandbox/src/local-stack-runtime.ts index 392d4dad..626632ac 100644 --- a/packages/sandbox/src/local-stack-runtime.ts +++ b/packages/sandbox/src/local-stack-runtime.ts @@ -38,14 +38,10 @@ const STACK_CONFIG_RETRY_MS = 2_000; */ export interface LocalStackRuntimeOptions { /** - * Supabase CLI version baked into the sandbox image (pinned default). - * Either an exact version (e.g. `2.109.1`) or a channel tag (`'stable'` | - * `'beta'`) that tracks npm's dist-tag for the `supabase` package — a - * channel tag's resolution is memoised for the lifetime of the process - * (see resolveCliVersion), not per session, so a later session in the same - * run will not pick up a newly published version. An eval's own - * `cliVersion:` frontmatter pin always wins over this option, whether this - * is an exact version or a channel tag. + * Supabase CLI version baked into the sandbox image: an exact version + * (e.g. `2.109.1`) or a channel tag (`'stable'` | `'beta'`) resolved + * against npm's dist-tag and memoised per process. An eval's own + * `cliVersion:` frontmatter pin always wins over this option. */ cliVersion?: CliChannel | (string & {}); /** From 945d77b692ae52d6e2c76a8702ab9c1aac89473f Mon Sep 17 00:00:00 2001 From: Colum Ferry Date: Tue, 22 Sep 2026 16:45:53 +0100 Subject: [PATCH 4/6] fix(sandbox): treat GitHub HEAD errors as failures and pin CLI channels once per run debAssetHeadOk conflated a GitHub 429/5xx with a missing asset (response.ok false for both), which could downgrade a beta release during a GitHub outage and record results under the wrong version. Only a 404 now means "asset not published"; any other non-2xx throws. The beta walk-back loop is updated to match: a 404 still advances to the next candidate, but a thrown error aborts the walk-back and propagates instead of being caught and skipped. Separately, nothing set SUPABASE_CLI_STABLE_VERSION/SUPABASE_CLI_BETA_VERSION, so each sandbox job resolved its own CLI channel version independently, risking two different versions in one fan-out if a release landed mid-run. runPairs now resolves each channel pin once via resolveChannelPins() before runBounded and threads the same pins into every job's .env write. --- .../scripts/run-vercel-evals.test.ts | 93 ++++++++++++++++++- apps/framework/scripts/run-vercel-evals.ts | 41 +++++++- packages/sandbox/src/cli-channel.ts | 20 ++-- packages/sandbox/test/cli-channel.test.ts | 91 +++++++++++++++--- 4 files changed, 217 insertions(+), 28 deletions(-) diff --git a/apps/framework/scripts/run-vercel-evals.test.ts b/apps/framework/scripts/run-vercel-evals.test.ts index 7da08d92..2161f956 100644 --- a/apps/framework/scripts/run-vercel-evals.test.ts +++ b/apps/framework/scripts/run-vercel-evals.test.ts @@ -1,4 +1,5 @@ import { APIError } from '@vercel/sandbox'; +import { resolveCliVersion } from '@supabase-evals/sandbox'; import { execFileSync } from 'node:child_process'; import { copyFileSync, @@ -11,7 +12,7 @@ import { } from 'node:fs'; import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; -import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { agentEnvironment, cleanupSandbox, @@ -22,11 +23,14 @@ import { isTerminalSandboxCreateError, packWorkspaceScript, parsePairs, + resolveChannelPins, runBounded, tagValue, expandJobs, } from './run-vercel-evals.js'; +vi.mock('@supabase-evals/sandbox', () => ({ resolveCliVersion: vi.fn() })); + describe('agentEnvironment', () => { const originalValues = new Map(); @@ -63,6 +67,93 @@ describe('agentEnvironment', () => { expect(env).not.toContain('SUPABASE_CLI_STABLE_VERSION'); expect(env).not.toContain('SUPABASE_CLI_BETA_VERSION'); }); + + it('prefers an explicit pin over the same-named process.env value', () => { + process.env.SUPABASE_CLI_STABLE_VERSION = 'env-value'; + + const env = agentEnvironment({ + SUPABASE_CLI_STABLE_VERSION: 'pinned-value', + }); + + expect(env).toContain('SUPABASE_CLI_STABLE_VERSION=pinned-value'); + expect(env).not.toContain('env-value'); + }); + + it('shares one pin value across multiple .env writes, simulating a two-job fan-out', () => { + const pins = { + SUPABASE_CLI_STABLE_VERSION: '2.117.0', + SUPABASE_CLI_BETA_VERSION: '2.118.0-beta.5', + }; + + const jobOneEnv = agentEnvironment(pins); + const jobTwoEnv = agentEnvironment(pins); + + expect(jobOneEnv).toBe(jobTwoEnv); + expect(jobOneEnv).toContain('SUPABASE_CLI_STABLE_VERSION=2.117.0'); + expect(jobOneEnv).toContain('SUPABASE_CLI_BETA_VERSION=2.118.0-beta.5'); + }); +}); + +describe('resolveChannelPins', () => { + const STABLE_ENV = 'SUPABASE_CLI_STABLE_VERSION'; + const BETA_ENV = 'SUPABASE_CLI_BETA_VERSION'; + const originalValues = new Map(); + + beforeEach(() => { + for (const name of [STABLE_ENV, BETA_ENV]) { + originalValues.set(name, process.env[name]); + delete process.env[name]; + } + vi.mocked(resolveCliVersion).mockReset(); + }); + + afterEach(() => { + for (const [name, value] of originalValues) { + if (value === undefined) delete process.env[name]; + else process.env[name] = value; + } + }); + + it('resolves each channel once and returns a pin every job can share', async () => { + vi.mocked(resolveCliVersion).mockImplementation(async (channel) => + channel === 'stable' ? '2.117.0' : '2.118.0-beta.5' + ); + + const pins = await resolveChannelPins(); + + expect(resolveCliVersion).toHaveBeenCalledTimes(2); + expect(resolveCliVersion).toHaveBeenCalledWith('stable'); + expect(resolveCliVersion).toHaveBeenCalledWith('beta'); + + // Two fanned-out jobs writing their own .env from the same pins object + // must get the identical value the resolver was called once for. + const jobOneEnv = agentEnvironment(pins); + const jobTwoEnv = agentEnvironment(pins); + expect(jobOneEnv).toBe(jobTwoEnv); + expect(jobOneEnv).toContain(`${STABLE_ENV}=2.117.0`); + expect(jobOneEnv).toContain(`${BETA_ENV}=2.118.0-beta.5`); + }); + + it('uses an already-set env var verbatim without calling the resolver', async () => { + process.env[STABLE_ENV] = '9.9.9'; + vi.mocked(resolveCliVersion).mockImplementation( + async () => '2.118.0-beta.5' + ); + + const pins = await resolveChannelPins(); + + expect(pins[STABLE_ENV]).toBe('9.9.9'); + expect(resolveCliVersion).toHaveBeenCalledTimes(1); + expect(resolveCliVersion).toHaveBeenCalledWith('beta'); + }); + + it('propagates a resolution failure rather than swallowing it', async () => { + vi.mocked(resolveCliVersion).mockRejectedValue( + new Error('npm unreachable') + ); + + await expect(resolveChannelPins()).rejects.toThrow('npm unreachable'); + }); }); describe('Vercel eval controller', () => { diff --git a/apps/framework/scripts/run-vercel-evals.ts b/apps/framework/scripts/run-vercel-evals.ts index ee231c2b..e06666fa 100644 --- a/apps/framework/scripts/run-vercel-evals.ts +++ b/apps/framework/scripts/run-vercel-evals.ts @@ -6,6 +6,7 @@ import { sandboxUsageSchema, type SandboxUsage, } from '@supabase-evals/core/eval-metadata'; +import { resolveCliVersion, type CliChannel } from '@supabase-evals/sandbox'; import { execFileSync } from 'node:child_process'; import { readFileSync, renameSync, writeFileSync } from 'node:fs'; import { join, resolve } from 'node:path'; @@ -28,6 +29,10 @@ export const FORWARDED_ENV_NAMES = [ 'SUPABASE_CLI_STABLE_VERSION', 'SUPABASE_CLI_BETA_VERSION', ]; +const CLI_CHANNEL_ENV: Record = { + stable: 'SUPABASE_CLI_STABLE_VERSION', + beta: 'SUPABASE_CLI_BETA_VERSION', +}; /** * Slack for the non-agent work inside `pnpm eval` (supabase start, resets, * scoring, export). Cold image pulls alone can take ~10 min. @@ -75,6 +80,29 @@ interface PairOptions extends RunnerOptions { pair: EvalPair; run: number; attempt: number; + pins: Record; +} + +/** + * Resolves each CLI channel's pin once, so every sandbox job in a fan-out + * runs against the same concrete version rather than each resolving + * independently and risking a mid-run release landing between them. An + * already-set env var is used verbatim, matching the workflow/manual + * override path in resolveCliVersion. + */ +export async function resolveChannelPins(): Promise> { + const resolved = await Promise.all( + (Object.entries(CLI_CHANNEL_ENV) as [CliChannel, string][]).map( + async ([channel, envVar]) => { + const override = process.env[envVar]; + return [ + envVar, + override ?? (await resolveCliVersion(channel)), + ] as const; + } + ) + ); + return Object.fromEntries(resolved); } interface SandboxCommandOptions { @@ -113,6 +141,10 @@ async function runPairs(options: RunnerOptions): Promise { `max ${options.concurrency} at a time` ); + // Resolved once so every job below writes the same pin, rather than each + // sandbox resolving its own channel version independently. + const pins = await resolveChannelPins(); + const results = await runBounded( jobs, options.concurrency, @@ -126,6 +158,7 @@ async function runPairs(options: RunnerOptions): Promise { pair, run, attempt, + pins, }, credentials ), @@ -269,7 +302,7 @@ async function runPairOnce( await sandbox.writeFiles([ { path: '.env', - content: `${agentEnvironment()}\n`, + content: `${agentEnvironment(options.pins)}\n`, }, ]); console.log(`${label} run eval`); @@ -625,11 +658,11 @@ function vercelCredentialsFromEnv(): { }; } -/** Serializes configured provider keys and CLI channel pins into the sandbox's `.env` file. */ -export function agentEnvironment(): string { +/** Serializes configured provider keys and CLI channel pins into the sandbox's `.env` file, preferring an explicit pin over the same-named process.env value. */ +export function agentEnvironment(pins: Record = {}): string { const lines: string[] = []; for (const name of FORWARDED_ENV_NAMES) { - const value = process.env[name]; + const value = pins[name] ?? process.env[name]; if (value) lines.push(`${name}=${value}`); } return lines.join('\n'); diff --git a/packages/sandbox/src/cli-channel.ts b/packages/sandbox/src/cli-channel.ts index 4990a4e0..7bc45c67 100644 --- a/packages/sandbox/src/cli-channel.ts +++ b/packages/sandbox/src/cli-channel.ts @@ -55,13 +55,16 @@ function releaseTagUrl(version: string): string { return `https://github.com/supabase/cli/releases/tag/v${version}`; } +/** Only a 404 means the asset is absent; any other non-2xx (e.g. a GitHub 429/5xx) is an error, not a missing-asset signal. */ async function debAssetHeadOk(url: string): Promise { const response = await fetch(url, { method: 'HEAD', redirect: 'follow', signal: AbortSignal.timeout(15_000), }); - return response.ok; + if (response.ok) return true; + if (response.status === 404) return false; + throw new Error(`HEAD ${url} -> ${response.status} ${response.statusText}`); } /** HEAD-checks both the amd64 and arm64 `.deb` assets, since the resolver's host architecture need not match the sandbox's. */ @@ -152,8 +155,10 @@ async function resolveCliVersionUncached(channel: CliChannel): Promise { /** * Walks back through published `X.Y.Z-beta.N` versions strictly older than * `unpublishedVersion`, newest first, for one with a downloadable `.deb`. - * Newer candidates are skipped too (likelier to be drafts); a transient - * probe error is logged and skipped rather than aborting the walk-back. + * Newer candidates are skipped too (likelier to be drafts). Only a 404 + * advances to the next candidate; any other probe error aborts the + * walk-back, since it means GitHub is unhealthy rather than that the + * candidate is absent. */ async function resolveFallbackBetaVersion( unpublishedVersion: string @@ -194,14 +199,7 @@ async function resolveFallbackBetaVersion( .slice(0, MAX_BETA_FALLBACK_CANDIDATES); for (const candidate of candidates) { - try { - if (await debAssetExists(candidate)) return candidate; - } catch (error) { - console.warn( - `[cli-channel] beta fallback candidate ${candidate} could not be checked, skipping: ` + - (error instanceof Error ? error.message : String(error)) - ); - } + if (await debAssetExists(candidate)) return candidate; } throw new Error( diff --git a/packages/sandbox/test/cli-channel.test.ts b/packages/sandbox/test/cli-channel.test.ts index 8bc1353a..95a34cbd 100644 --- a/packages/sandbox/test/cli-channel.test.ts +++ b/packages/sandbox/test/cli-channel.test.ts @@ -19,8 +19,12 @@ function jsonResponse( }; } -function headResponse(ok: boolean) { - return { ok, status: ok ? 200 : 404, statusText: ok ? 'OK' : 'Not Found' }; +function headResponse(ok: boolean, status?: number, statusText?: string) { + return { + ok, + status: status ?? (ok ? 200 : 404), + statusText: statusText ?? (ok ? 'OK' : 'Not Found'), + }; } /** @@ -350,12 +354,13 @@ describe('resolveCliVersion', () => { await expect(resolveCliVersion('beta')).resolves.toBe('2.118.0-beta.49'); }); - it('treats a walk-back candidate whose asset check throws as unavailable and continues to the next', async () => { - const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}); + it('aborts the walk-back and propagates when a candidate probe throws, rather than continuing to the next candidate', async () => { const fetchMock = vi.fn(async (url: string, init?: RequestInit) => { if (init?.method === 'HEAD') { if (url.includes('2.118.0-beta.52')) return headResponse(false); - if (url.includes('2.118.0-beta.51')) throw new Error('network blip'); + if (url.includes('2.118.0-beta.51')) { + return headResponse(false, 500, 'Internal Server Error'); + } if (url.includes('2.118.0-beta.50')) return headResponse(true); return headResponse(false); } @@ -376,15 +381,50 @@ describe('resolveCliVersion', () => { vi.stubGlobal('fetch', fetchMock); const { resolveCliVersion } = await import('../src/cli-channel.js'); - await expect(resolveCliVersion('beta')).resolves.toBe('2.118.0-beta.50'); - expect(warnSpy).toHaveBeenCalledWith( - expect.stringContaining('2.118.0-beta.51') - ); - expect(warnSpy).toHaveBeenCalledWith( - expect.stringContaining('network blip') + await expect(resolveCliVersion('beta')).rejects.toThrow( + /2\.118\.0-beta\.51.*-> 500 Internal Server Error/ ); - warnSpy.mockRestore(); + const headUrls = fetchMock.mock.calls + .filter(([, init]) => init?.method === 'HEAD') + .map(([url]) => url); + // beta.50 is never probed — the throw on beta.51 aborts the walk-back. + expect(headUrls).toEqual([ + cliDebUrl('2.118.0-beta.52', 'amd64'), + cliDebUrl('2.118.0-beta.52', 'arm64'), + cliDebUrl('2.118.0-beta.51', 'amd64'), + cliDebUrl('2.118.0-beta.51', 'arm64'), + ]); + }); + + it('advances to the second walk-back candidate when the first is a 404', async () => { + const fetchMock = routedFetchMock({ + distTags: { latest: '1.2.3', beta: '2.118.0-beta.52' }, + assetOk: (version) => version === '2.118.0-beta.50', + packument: { + versions: { + '2.118.0-beta.52': {}, + '2.118.0-beta.51': {}, + '2.118.0-beta.50': {}, + }, + }, + }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersion } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersion('beta')).resolves.toBe('2.118.0-beta.50'); + + const headUrls = fetchMock.mock.calls + .filter(([, init]) => init?.method === 'HEAD') + .map(([url]) => url); + expect(headUrls).toEqual([ + cliDebUrl('2.118.0-beta.52', 'amd64'), + cliDebUrl('2.118.0-beta.52', 'arm64'), + cliDebUrl('2.118.0-beta.51', 'amd64'), + cliDebUrl('2.118.0-beta.51', 'arm64'), + cliDebUrl('2.118.0-beta.50', 'amd64'), + cliDebUrl('2.118.0-beta.50', 'arm64'), + ]); }); it('caps the beta walk-back at MAX_BETA_FALLBACK_CANDIDATES, probing newest-first', async () => { @@ -453,6 +493,33 @@ describe('resolveCliVersion', () => { ); }); + it('throws naming the URL and status when the dist-tag asset HEAD is a GitHub error, without fetching the packument', async () => { + const fetchMock = vi.fn(async (url: string, init?: RequestInit) => { + if (init?.method === 'HEAD') { + return url.includes('_amd64') + ? headResponse(false, 500, 'Internal Server Error') + : headResponse(true); + } + if (url === DIST_TAGS_URL) { + return jsonResponse({ latest: '1.2.3', beta: '2.118.0-beta.52' }); + } + if (url === PACKUMENT_URL) { + throw new Error('packument must not be fetched'); + } + throw new Error(`unexpected fetch: ${url}`); + }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersion } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersion('beta')).rejects.toThrow( + `${cliDebUrl('2.118.0-beta.52', 'amd64')} -> 500 Internal Server Error` + ); + + expect(fetchMock.mock.calls.some(([url]) => url === PACKUMENT_URL)).toBe( + false + ); + }); + it('does not hit the network at all for an env override', async () => { process.env[STABLE_ENV] = '9.9.9'; process.env[BETA_ENV] = '9.9.9-beta.1'; From 18f83928b0ce0215959458d0292da7c051219b1e Mon Sep 17 00:00:00 2001 From: Colum Ferry Date: Tue, 22 Sep 2026 18:12:19 +0100 Subject: [PATCH 5/6] fix(sandbox): resolve only the CLI channels a run's pairs actually need Union each pair's effective channel (exact eval pins win and need none; otherwise the pair's experiment's localStack.cliChannel) before fan-out, instead of unconditionally resolving both channels. A run with no CLI-channel experiments now does zero npm/GitHub lookups. An experiment that can't be resolved to a config throws instead of silently counting as needing no channel. --- .../scripts/run-vercel-evals.test.ts | 113 +++++++++++++++++- apps/framework/scripts/run-vercel-evals.ts | 110 ++++++++++++++--- packages/core/src/index.ts | 2 + packages/sandbox/src/cli-channel.ts | 2 +- packages/sandbox/src/local-stack-runtime.ts | 10 +- 5 files changed, 212 insertions(+), 25 deletions(-) diff --git a/apps/framework/scripts/run-vercel-evals.test.ts b/apps/framework/scripts/run-vercel-evals.test.ts index 2161f956..ab4cec9a 100644 --- a/apps/framework/scripts/run-vercel-evals.test.ts +++ b/apps/framework/scripts/run-vercel-evals.test.ts @@ -1,5 +1,5 @@ import { APIError } from '@vercel/sandbox'; -import { resolveCliVersion } from '@supabase-evals/sandbox'; +import { resolveCliVersion, type CliChannel } from '@supabase-evals/sandbox'; import { execFileSync } from 'node:child_process'; import { copyFileSync, @@ -23,10 +23,12 @@ import { isTerminalSandboxCreateError, packWorkspaceScript, parsePairs, + requiredCliChannels, resolveChannelPins, runBounded, tagValue, expandJobs, + type EvalPair, } from './run-vercel-evals.js'; vi.mock('@supabase-evals/sandbox', () => ({ resolveCliVersion: vi.fn() })); @@ -97,6 +99,7 @@ describe('agentEnvironment', () => { describe('resolveChannelPins', () => { const STABLE_ENV = 'SUPABASE_CLI_STABLE_VERSION'; const BETA_ENV = 'SUPABASE_CLI_BETA_VERSION'; + const BOTH_CHANNELS = new Set(['stable', 'beta']); const originalValues = new Map(); beforeEach(() => { @@ -114,12 +117,12 @@ describe('resolveChannelPins', () => { } }); - it('resolves each channel once and returns a pin every job can share', async () => { + it('resolves only the requested channels and returns a pin every job can share', async () => { vi.mocked(resolveCliVersion).mockImplementation(async (channel) => channel === 'stable' ? '2.117.0' : '2.118.0-beta.5' ); - const pins = await resolveChannelPins(); + const pins = await resolveChannelPins(BOTH_CHANNELS); expect(resolveCliVersion).toHaveBeenCalledTimes(2); expect(resolveCliVersion).toHaveBeenCalledWith('stable'); @@ -134,13 +137,32 @@ describe('resolveChannelPins', () => { expect(jobOneEnv).toContain(`${BETA_ENV}=2.118.0-beta.5`); }); + it('resolves only stable when that is the only requested channel, never calling resolveCliVersion with beta', async () => { + vi.mocked(resolveCliVersion).mockImplementation(async (channel) => + channel === 'stable' ? '2.117.0' : '2.118.0-beta.5' + ); + + const pins = await resolveChannelPins(new Set(['stable'])); + + expect(pins).toEqual({ [STABLE_ENV]: '2.117.0' }); + expect(resolveCliVersion).toHaveBeenCalledTimes(1); + expect(resolveCliVersion).not.toHaveBeenCalledWith('beta'); + }); + + it('does no network work for an empty channel set', async () => { + const pins = await resolveChannelPins(new Set()); + + expect(pins).toEqual({}); + expect(resolveCliVersion).not.toHaveBeenCalled(); + }); + it('uses an already-set env var verbatim without calling the resolver', async () => { process.env[STABLE_ENV] = '9.9.9'; vi.mocked(resolveCliVersion).mockImplementation( async () => '2.118.0-beta.5' ); - const pins = await resolveChannelPins(); + const pins = await resolveChannelPins(BOTH_CHANNELS); expect(pins[STABLE_ENV]).toBe('9.9.9'); expect(resolveCliVersion).toHaveBeenCalledTimes(1); @@ -152,7 +174,88 @@ describe('resolveChannelPins', () => { new Error('npm unreachable') ); - await expect(resolveChannelPins()).rejects.toThrow('npm unreachable'); + await expect( + resolveChannelPins(new Set(['stable'])) + ).rejects.toThrow('npm unreachable'); + }); +}); + +describe('requiredCliChannels', () => { + const pair = (overrides: Partial = {}): EvalPair => ({ + eval_id: 'eval-1', + experiment: 'experiment-1', + experiment_suite: 'benchmark', + eval_suite: 'benchmark', + ...overrides, + }); + + it('resolves only the channel a stable-tagged experiment declares', async () => { + const loadExperimentConfig = vi.fn(async () => ({ + localStack: { cliChannel: 'stable' as const }, + })); + + const channels = await requiredCliChannels([pair()], { + loadEvalMetadata: () => ({ cliVersion: undefined }), + loadExperimentConfig, + }); + + expect(channels).toEqual(new Set(['stable'])); + }); + + it('resolves nothing when no experiment in the pair set declares a channel', async () => { + const loadExperimentConfig = vi.fn(async () => ({})); + + const channels = await requiredCliChannels( + [pair(), pair({ eval_id: 'eval-2', experiment: 'experiment-2' })], + { + loadEvalMetadata: () => ({ cliVersion: undefined }), + loadExperimentConfig, + } + ); + + expect(channels.size).toBe(0); + }); + + it("an eval's pinned cliVersion contributes no channel, even when its experiment declares one", async () => { + const loadExperimentConfig = vi.fn(async () => ({ + localStack: { cliChannel: 'beta' as const }, + })); + + const channels = await requiredCliChannels([pair()], { + loadEvalMetadata: () => ({ cliVersion: '2.109.1' }), + loadExperimentConfig, + }); + + expect(channels.size).toBe(0); + expect(loadExperimentConfig).not.toHaveBeenCalled(); + }); + + it('unions channels across pairs without resolving an experiment config twice', async () => { + const loadExperimentConfig = vi.fn(async () => ({ + localStack: { cliChannel: 'beta' as const }, + })); + + const channels = await requiredCliChannels( + [pair(), pair({ eval_id: 'eval-2' })], + { + loadEvalMetadata: () => ({ cliVersion: undefined }), + loadExperimentConfig, + } + ); + + expect(channels).toEqual(new Set(['beta'])); + expect(loadExperimentConfig).toHaveBeenCalledTimes(1); + }); + + it('throws naming an experiment that cannot be resolved to a config', async () => { + await expect( + requiredCliChannels([pair({ experiment: 'ghost' })], { + loadEvalMetadata: () => ({ cliVersion: undefined }), + loadExperimentConfig: async () => { + throw new Error('no experiment config found for "ghost"'); + }, + }) + ).rejects.toThrow('no experiment config found for "ghost"'); }); }); diff --git a/apps/framework/scripts/run-vercel-evals.ts b/apps/framework/scripts/run-vercel-evals.ts index e06666fa..8a58c553 100644 --- a/apps/framework/scripts/run-vercel-evals.ts +++ b/apps/framework/scripts/run-vercel-evals.ts @@ -1,6 +1,8 @@ #!/usr/bin/env tsx import { APIError, Sandbox } from '@vercel/sandbox'; +import type { EvalMetadata, ExperimentConfig } from '@supabase-evals/core'; +import { parseEvalMarkdown } from '@supabase-evals/core/eval-markdown'; import { rawEvalResultSchema, sandboxUsageSchema, @@ -8,8 +10,8 @@ import { } from '@supabase-evals/core/eval-metadata'; import { resolveCliVersion, type CliChannel } from '@supabase-evals/sandbox'; import { execFileSync } from 'node:child_process'; -import { readFileSync, renameSync, writeFileSync } from 'node:fs'; -import { join, resolve } from 'node:path'; +import { existsSync, readFileSync, renameSync, writeFileSync } from 'node:fs'; +import { join, relative, resolve } from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; import pLimit from 'p-limit'; import pRetry, { AbortError } from 'p-retry'; @@ -84,27 +86,94 @@ interface PairOptions extends RunnerOptions { } /** - * Resolves each CLI channel's pin once, so every sandbox job in a fan-out - * runs against the same concrete version rather than each resolving + * Resolves each requested CLI channel's pin once, so every sandbox job in a + * fan-out runs against the same concrete version rather than each resolving * independently and risking a mid-run release landing between them. An * already-set env var is used verbatim, matching the workflow/manual - * override path in resolveCliVersion. + * override path in resolveCliVersion. An empty set does no network work. */ -export async function resolveChannelPins(): Promise> { +export async function resolveChannelPins( + channels: ReadonlySet +): Promise> { const resolved = await Promise.all( - (Object.entries(CLI_CHANNEL_ENV) as [CliChannel, string][]).map( - async ([channel, envVar]) => { - const override = process.env[envVar]; - return [ - envVar, - override ?? (await resolveCliVersion(channel)), - ] as const; - } - ) + [...channels].map(async (channel) => { + const envVar = CLI_CHANNEL_ENV[channel]; + const override = process.env[envVar]; + return [envVar, override ?? (await resolveCliVersion(channel))] as const; + }) ); return Object.fromEntries(resolved); } +export interface RequiredCliChannelsDeps { + loadEvalMetadata: (pair: EvalPair) => Pick; + loadExperimentConfig: ( + experiment: string + ) => Promise<{ localStack?: { cliChannel?: CliChannel } }>; +} + +/** + * Maps a pair set to the CLI channels at least one pair needs, so + * resolveChannelPins only resolves those. An eval's own `cliVersion` + * frontmatter is an exact pin that wins over its experiment's channel and + * needs no resolution; a pair whose experiment has no `localStack.cliChannel` + * needs none either. + */ +export async function requiredCliChannels( + pairs: readonly EvalPair[], + { loadEvalMetadata, loadExperimentConfig }: RequiredCliChannelsDeps +): Promise> { + const channels = new Set(); + const configs = new Map>(); + + for (const pair of pairs) { + if (loadEvalMetadata(pair).cliVersion !== undefined) continue; + + let config = configs.get(pair.experiment); + if (!config) { + config = loadExperimentConfig(pair.experiment); + configs.set(pair.experiment, config); + } + const channel = (await config).localStack?.cliChannel; + if (channel) channels.add(channel); + } + + return channels; +} + +/** Mirrors run-eval.ts's evals///PROMPT.md convention. */ +function loadEvalMetadata(pair: EvalPair): EvalMetadata { + const promptPath = join( + ROOT, + 'evals', + pair.eval_suite, + pair.eval_id, + 'PROMPT.md' + ); + return parseEvalMarkdown(readFileSync(promptPath, 'utf8'), promptPath) + .metadata; +} + +/** + * Mirrors run-eval.ts's loadExperiments(), which resolves an experiment name + * to experiments/.ts. Throws rather than treating a config it can't + * find as needing no channel, so a future change to this layout (e.g. a + * nested experiments//*.experiment.ts convention) fails loudly instead + * of silently resolving every channel per-sandbox again. + */ +async function loadExperimentConfig( + experiment: string +): Promise { + const path = join(ROOT, 'experiments', `${experiment}.ts`); + if (!existsSync(path)) { + throw new Error( + `no experiment config found for "${experiment}" at ${relative(ROOT, path)}` + ); + } + const mod = await import(pathToFileURL(path).href); + return mod.default as ExperimentConfig; +} + interface SandboxCommandOptions { cmd: string; args?: string[]; @@ -141,9 +210,14 @@ async function runPairs(options: RunnerOptions): Promise { `max ${options.concurrency} at a time` ); - // Resolved once so every job below writes the same pin, rather than each - // sandbox resolving its own channel version independently. - const pins = await resolveChannelPins(); + // Resolved once, for only the channels this run's pairs actually need, so + // every job below writes the same pin rather than each sandbox resolving + // its own channel version independently. + const channels = await requiredCliChannels(options.pairs, { + loadEvalMetadata, + loadExperimentConfig, + }); + const pins = await resolveChannelPins(channels); const results = await runBounded( jobs, diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index f2550a1c..311fca73 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -582,6 +582,8 @@ export type LocalStackSession = { export type LocalStackRuntime = { id: string; startSession(args: LocalStackSessionArgs): Promise; + /** Channel (e.g. `beta`) this runtime resolves its CLI version against, unset when it's pinned to an exact version. */ + cliChannel?: 'stable' | 'beta'; }; export type ExperimentConfig = { diff --git a/packages/sandbox/src/cli-channel.ts b/packages/sandbox/src/cli-channel.ts index 7bc45c67..26167b84 100644 --- a/packages/sandbox/src/cli-channel.ts +++ b/packages/sandbox/src/cli-channel.ts @@ -41,7 +41,7 @@ const versionCache = new Map>(); const CLI_CHANNELS = new Set(['stable', 'beta']); -function isCliChannel(value: string): value is CliChannel { +export function isCliChannel(value: string): value is CliChannel { return CLI_CHANNELS.has(value as CliChannel); } diff --git a/packages/sandbox/src/local-stack-runtime.ts b/packages/sandbox/src/local-stack-runtime.ts index 626632ac..08ba4a71 100644 --- a/packages/sandbox/src/local-stack-runtime.ts +++ b/packages/sandbox/src/local-stack-runtime.ts @@ -15,7 +15,11 @@ import { DockerSandbox } from './docker-sandbox.js'; import { createAgentEnvironment } from './agent-environment.js'; import { ensureEdgeRuntime, teardownSupabaseProject } from './supabase.js'; import { buildSkillsPrompt } from './skills.js'; -import { resolveCliVersionOption, type CliChannel } from './cli-channel.js'; +import { + isCliChannel, + resolveCliVersionOption, + type CliChannel, +} from './cli-channel.js'; const DEFAULT_BASH_TIMEOUT_SEC = 240; const MAX_BASH_TIMEOUT_SEC = 600; @@ -80,6 +84,10 @@ export function localStackRuntime( ): LocalStackRuntime { return { id: 'local-stack', + cliChannel: + options.cliVersion !== undefined && isCliChannel(options.cliVersion) + ? options.cliVersion + : undefined, async startSession({ agent, cliVersion, From 0a794b27abe02e554f5dab9d96e078fa9dce9f1a Mon Sep 17 00:00:00 2001 From: Colum Ferry Date: Tue, 22 Sep 2026 19:55:10 +0100 Subject: [PATCH 6/6] fix(sandbox): resolve experiment configs through discoverExperimentFiles The controller's channel lookup built experiments/.ts by hand, which stopped matching when experiments moved to experiments//*.experiment.ts. Use the shared discovery helper so the path convention lives in one place. --- apps/framework/scripts/run-vercel-evals.ts | 27 ++++++++++++++-------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/apps/framework/scripts/run-vercel-evals.ts b/apps/framework/scripts/run-vercel-evals.ts index 8a58c553..8de4ba29 100644 --- a/apps/framework/scripts/run-vercel-evals.ts +++ b/apps/framework/scripts/run-vercel-evals.ts @@ -10,13 +10,14 @@ import { } from '@supabase-evals/core/eval-metadata'; import { resolveCliVersion, type CliChannel } from '@supabase-evals/sandbox'; import { execFileSync } from 'node:child_process'; -import { existsSync, readFileSync, renameSync, writeFileSync } from 'node:fs'; +import { readFileSync, renameSync, writeFileSync } from 'node:fs'; import { join, relative, resolve } from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; import pLimit from 'p-limit'; import pRetry, { AbortError } from 'p-retry'; import { z } from 'zod'; import { positiveInteger, readFlag } from '../lib/cli-args.js'; +import { discoverExperimentFiles } from '../lib/experiment-files.js'; const ROOT = fileURLToPath(new URL('../../../', import.meta.url)); /** Base for sandbox URLs printed during runs */ @@ -154,20 +155,28 @@ function loadEvalMetadata(pair: EvalPair): EvalMetadata { .metadata; } +const EXPERIMENTS_DIR = join(ROOT, 'experiments'); + +let experimentPaths: Promise> | undefined; + +function experimentPathsByName(): Promise> { + experimentPaths ??= discoverExperimentFiles(EXPERIMENTS_DIR).then( + (files) => new Map(files.map((file) => [file.name, file.path])) + ); + return experimentPaths; +} + /** - * Mirrors run-eval.ts's loadExperiments(), which resolves an experiment name - * to experiments/.ts. Throws rather than treating a config it can't - * find as needing no channel, so a future change to this layout (e.g. a - * nested experiments//*.experiment.ts convention) fails loudly instead - * of silently resolving every channel per-sandbox again. + * Throws rather than treating a config it can't find as needing no channel, + * which would silently resolve every channel per-sandbox again. */ async function loadExperimentConfig( experiment: string ): Promise { - const path = join(ROOT, 'experiments', `${experiment}.ts`); - if (!existsSync(path)) { + const path = (await experimentPathsByName()).get(experiment); + if (!path) { throw new Error( - `no experiment config found for "${experiment}" at ${relative(ROOT, path)}` + `no experiment config found for "${experiment}" under ${relative(ROOT, EXPERIMENTS_DIR)}` ); } const mod = await import(pathToFileURL(path).href);