diff --git a/README.md b/README.md index ceb81932..62594965 100644 --- a/README.md +++ b/README.md @@ -145,6 +145,8 @@ An eval's optional `local/` directory is copied into the sandbox workspace befor Set `cliVersion: 2.109.1` in an eval's frontmatter when it requires a specific Supabase CLI release. This overrides an experiment's `localStackRuntime({ cliVersion })` setting; otherwise the runtime setting or repository-wide default applies. +An experiment can instead pass `localStackRuntime({ cliVersion: 'stable' })` or `'beta'` to track npm's dist-tag for the `supabase` package rather than an exact version, resolved once at session start. An eval's own `cliVersion:` pin still wins over either form. + Scorers check what the agent produced, never what the harness provisioned: with `projectRunning: true` (the default) the running stack and the seeded `local/` workspace are setup, so score only the deltas the agent made on top; with `projectRunning: false` the agent creates that state itself, so depending on it is fair game. Test the sandbox plumbing without an agent run (Docker required, not part of `pnpm check`): diff --git a/apps/framework/scripts/run-vercel-evals.test.ts b/apps/framework/scripts/run-vercel-evals.test.ts index 1190490a..ab4cec9a 100644 --- a/apps/framework/scripts/run-vercel-evals.test.ts +++ b/apps/framework/scripts/run-vercel-evals.test.ts @@ -1,4 +1,5 @@ import { APIError } from '@vercel/sandbox'; +import { resolveCliVersion, type CliChannel } from '@supabase-evals/sandbox'; import { execFileSync } from 'node:child_process'; import { copyFileSync, @@ -11,20 +12,253 @@ import { } from 'node:fs'; import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; -import { describe, expect, it } from 'vitest'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { + agentEnvironment, cleanupSandbox, downloadResults, finalizeResult, + FORWARDED_ENV_NAMES, isRetryableSandboxCreateError, isTerminalSandboxCreateError, packWorkspaceScript, parsePairs, + requiredCliChannels, + resolveChannelPins, runBounded, tagValue, expandJobs, + type EvalPair, } from './run-vercel-evals.js'; +vi.mock('@supabase-evals/sandbox', () => ({ resolveCliVersion: vi.fn() })); + +describe('agentEnvironment', () => { + const originalValues = new Map(); + + beforeEach(() => { + for (const name of FORWARDED_ENV_NAMES) { + originalValues.set(name, process.env[name]); + delete process.env[name]; + } + }); + + afterEach(() => { + for (const [name, value] of originalValues) { + if (value === undefined) delete process.env[name]; + else process.env[name] = value; + } + }); + + it('forwards every configured name when set', () => { + for (const name of FORWARDED_ENV_NAMES) { + process.env[name] = `${name}-value`; + } + + const lines = agentEnvironment().split('\n'); + for (const name of FORWARDED_ENV_NAMES) { + expect(lines).toContain(`${name}=${name}-value`); + } + }); + + it('omits the CLI channel pins when unset', () => { + process.env.ANTHROPIC_API_KEY = 'anthropic-value'; + + const env = agentEnvironment(); + expect(env).toBe('ANTHROPIC_API_KEY=anthropic-value'); + expect(env).not.toContain('SUPABASE_CLI_STABLE_VERSION'); + expect(env).not.toContain('SUPABASE_CLI_BETA_VERSION'); + }); + + it('prefers an explicit pin over the same-named process.env value', () => { + process.env.SUPABASE_CLI_STABLE_VERSION = 'env-value'; + + const env = agentEnvironment({ + SUPABASE_CLI_STABLE_VERSION: 'pinned-value', + }); + + expect(env).toContain('SUPABASE_CLI_STABLE_VERSION=pinned-value'); + expect(env).not.toContain('env-value'); + }); + + it('shares one pin value across multiple .env writes, simulating a two-job fan-out', () => { + const pins = { + SUPABASE_CLI_STABLE_VERSION: '2.117.0', + SUPABASE_CLI_BETA_VERSION: '2.118.0-beta.5', + }; + + const jobOneEnv = agentEnvironment(pins); + const jobTwoEnv = agentEnvironment(pins); + + expect(jobOneEnv).toBe(jobTwoEnv); + expect(jobOneEnv).toContain('SUPABASE_CLI_STABLE_VERSION=2.117.0'); + expect(jobOneEnv).toContain('SUPABASE_CLI_BETA_VERSION=2.118.0-beta.5'); + }); +}); + +describe('resolveChannelPins', () => { + const STABLE_ENV = 'SUPABASE_CLI_STABLE_VERSION'; + const BETA_ENV = 'SUPABASE_CLI_BETA_VERSION'; + const BOTH_CHANNELS = new Set(['stable', 'beta']); + const originalValues = new Map(); + + beforeEach(() => { + for (const name of [STABLE_ENV, BETA_ENV]) { + originalValues.set(name, process.env[name]); + delete process.env[name]; + } + vi.mocked(resolveCliVersion).mockReset(); + }); + + afterEach(() => { + for (const [name, value] of originalValues) { + if (value === undefined) delete process.env[name]; + else process.env[name] = value; + } + }); + + it('resolves only the requested channels and returns a pin every job can share', async () => { + vi.mocked(resolveCliVersion).mockImplementation(async (channel) => + channel === 'stable' ? '2.117.0' : '2.118.0-beta.5' + ); + + const pins = await resolveChannelPins(BOTH_CHANNELS); + + expect(resolveCliVersion).toHaveBeenCalledTimes(2); + expect(resolveCliVersion).toHaveBeenCalledWith('stable'); + expect(resolveCliVersion).toHaveBeenCalledWith('beta'); + + // Two fanned-out jobs writing their own .env from the same pins object + // must get the identical value the resolver was called once for. + const jobOneEnv = agentEnvironment(pins); + const jobTwoEnv = agentEnvironment(pins); + expect(jobOneEnv).toBe(jobTwoEnv); + expect(jobOneEnv).toContain(`${STABLE_ENV}=2.117.0`); + expect(jobOneEnv).toContain(`${BETA_ENV}=2.118.0-beta.5`); + }); + + it('resolves only stable when that is the only requested channel, never calling resolveCliVersion with beta', async () => { + vi.mocked(resolveCliVersion).mockImplementation(async (channel) => + channel === 'stable' ? '2.117.0' : '2.118.0-beta.5' + ); + + const pins = await resolveChannelPins(new Set(['stable'])); + + expect(pins).toEqual({ [STABLE_ENV]: '2.117.0' }); + expect(resolveCliVersion).toHaveBeenCalledTimes(1); + expect(resolveCliVersion).not.toHaveBeenCalledWith('beta'); + }); + + it('does no network work for an empty channel set', async () => { + const pins = await resolveChannelPins(new Set()); + + expect(pins).toEqual({}); + expect(resolveCliVersion).not.toHaveBeenCalled(); + }); + + it('uses an already-set env var verbatim without calling the resolver', async () => { + process.env[STABLE_ENV] = '9.9.9'; + vi.mocked(resolveCliVersion).mockImplementation( + async () => '2.118.0-beta.5' + ); + + const pins = await resolveChannelPins(BOTH_CHANNELS); + + expect(pins[STABLE_ENV]).toBe('9.9.9'); + expect(resolveCliVersion).toHaveBeenCalledTimes(1); + expect(resolveCliVersion).toHaveBeenCalledWith('beta'); + }); + + it('propagates a resolution failure rather than swallowing it', async () => { + vi.mocked(resolveCliVersion).mockRejectedValue( + new Error('npm unreachable') + ); + + await expect( + resolveChannelPins(new Set(['stable'])) + ).rejects.toThrow('npm unreachable'); + }); +}); + +describe('requiredCliChannels', () => { + const pair = (overrides: Partial = {}): EvalPair => ({ + eval_id: 'eval-1', + experiment: 'experiment-1', + experiment_suite: 'benchmark', + eval_suite: 'benchmark', + ...overrides, + }); + + it('resolves only the channel a stable-tagged experiment declares', async () => { + const loadExperimentConfig = vi.fn(async () => ({ + localStack: { cliChannel: 'stable' as const }, + })); + + const channels = await requiredCliChannels([pair()], { + loadEvalMetadata: () => ({ cliVersion: undefined }), + loadExperimentConfig, + }); + + expect(channels).toEqual(new Set(['stable'])); + }); + + it('resolves nothing when no experiment in the pair set declares a channel', async () => { + const loadExperimentConfig = vi.fn(async () => ({})); + + const channels = await requiredCliChannels( + [pair(), pair({ eval_id: 'eval-2', experiment: 'experiment-2' })], + { + loadEvalMetadata: () => ({ cliVersion: undefined }), + loadExperimentConfig, + } + ); + + expect(channels.size).toBe(0); + }); + + it("an eval's pinned cliVersion contributes no channel, even when its experiment declares one", async () => { + const loadExperimentConfig = vi.fn(async () => ({ + localStack: { cliChannel: 'beta' as const }, + })); + + const channels = await requiredCliChannels([pair()], { + loadEvalMetadata: () => ({ cliVersion: '2.109.1' }), + loadExperimentConfig, + }); + + expect(channels.size).toBe(0); + expect(loadExperimentConfig).not.toHaveBeenCalled(); + }); + + it('unions channels across pairs without resolving an experiment config twice', async () => { + const loadExperimentConfig = vi.fn(async () => ({ + localStack: { cliChannel: 'beta' as const }, + })); + + const channels = await requiredCliChannels( + [pair(), pair({ eval_id: 'eval-2' })], + { + loadEvalMetadata: () => ({ cliVersion: undefined }), + loadExperimentConfig, + } + ); + + expect(channels).toEqual(new Set(['beta'])); + expect(loadExperimentConfig).toHaveBeenCalledTimes(1); + }); + + it('throws naming an experiment that cannot be resolved to a config', async () => { + await expect( + requiredCliChannels([pair({ experiment: 'ghost' })], { + loadEvalMetadata: () => ({ cliVersion: undefined }), + loadExperimentConfig: async () => { + throw new Error('no experiment config found for "ghost"'); + }, + }) + ).rejects.toThrow('no experiment config found for "ghost"'); + }); +}); + describe('Vercel eval controller', () => { it('bounds concurrent work and lets independent failures settle', async () => { let active = 0; diff --git a/apps/framework/scripts/run-vercel-evals.ts b/apps/framework/scripts/run-vercel-evals.ts index 221cdcaa..8de4ba29 100644 --- a/apps/framework/scripts/run-vercel-evals.ts +++ b/apps/framework/scripts/run-vercel-evals.ts @@ -1,30 +1,41 @@ #!/usr/bin/env tsx import { APIError, Sandbox } from '@vercel/sandbox'; +import type { EvalMetadata, ExperimentConfig } from '@supabase-evals/core'; +import { parseEvalMarkdown } from '@supabase-evals/core/eval-markdown'; import { rawEvalResultSchema, sandboxUsageSchema, type SandboxUsage, } from '@supabase-evals/core/eval-metadata'; +import { resolveCliVersion, type CliChannel } from '@supabase-evals/sandbox'; import { execFileSync } from 'node:child_process'; import { readFileSync, renameSync, writeFileSync } from 'node:fs'; -import { join, resolve } from 'node:path'; +import { join, relative, resolve } from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; import pLimit from 'p-limit'; import pRetry, { AbortError } from 'p-retry'; import { z } from 'zod'; import { positiveInteger, readFlag } from '../lib/cli-args.js'; +import { discoverExperimentFiles } from '../lib/experiment-files.js'; const ROOT = fileURLToPath(new URL('../../../', import.meta.url)); /** Base for sandbox URLs printed during runs */ const SANDBOX_DASHBOARD_URL = 'https://vercel.com/supabase/evals-runner/sandboxes'; -const AGENT_ENV_NAMES = [ +export const FORWARDED_ENV_NAMES = [ 'ANTHROPIC_API_KEY', 'OPENAI_API_KEY', 'AI_GATEWAY_API_KEY', 'XAI_API_KEY', + // Pins the CLI channel version resolved for this run across sandbox jobs. + 'SUPABASE_CLI_STABLE_VERSION', + 'SUPABASE_CLI_BETA_VERSION', ]; +const CLI_CHANNEL_ENV: Record = { + stable: 'SUPABASE_CLI_STABLE_VERSION', + beta: 'SUPABASE_CLI_BETA_VERSION', +}; /** * Slack for the non-agent work inside `pnpm eval` (supabase start, resets, * scoring, export). Cold image pulls alone can take ~10 min. @@ -72,6 +83,104 @@ interface PairOptions extends RunnerOptions { pair: EvalPair; run: number; attempt: number; + pins: Record; +} + +/** + * Resolves each requested CLI channel's pin once, so every sandbox job in a + * fan-out runs against the same concrete version rather than each resolving + * independently and risking a mid-run release landing between them. An + * already-set env var is used verbatim, matching the workflow/manual + * override path in resolveCliVersion. An empty set does no network work. + */ +export async function resolveChannelPins( + channels: ReadonlySet +): Promise> { + const resolved = await Promise.all( + [...channels].map(async (channel) => { + const envVar = CLI_CHANNEL_ENV[channel]; + const override = process.env[envVar]; + return [envVar, override ?? (await resolveCliVersion(channel))] as const; + }) + ); + return Object.fromEntries(resolved); +} + +export interface RequiredCliChannelsDeps { + loadEvalMetadata: (pair: EvalPair) => Pick; + loadExperimentConfig: ( + experiment: string + ) => Promise<{ localStack?: { cliChannel?: CliChannel } }>; +} + +/** + * Maps a pair set to the CLI channels at least one pair needs, so + * resolveChannelPins only resolves those. An eval's own `cliVersion` + * frontmatter is an exact pin that wins over its experiment's channel and + * needs no resolution; a pair whose experiment has no `localStack.cliChannel` + * needs none either. + */ +export async function requiredCliChannels( + pairs: readonly EvalPair[], + { loadEvalMetadata, loadExperimentConfig }: RequiredCliChannelsDeps +): Promise> { + const channels = new Set(); + const configs = new Map>(); + + for (const pair of pairs) { + if (loadEvalMetadata(pair).cliVersion !== undefined) continue; + + let config = configs.get(pair.experiment); + if (!config) { + config = loadExperimentConfig(pair.experiment); + configs.set(pair.experiment, config); + } + const channel = (await config).localStack?.cliChannel; + if (channel) channels.add(channel); + } + + return channels; +} + +/** Mirrors run-eval.ts's evals///PROMPT.md convention. */ +function loadEvalMetadata(pair: EvalPair): EvalMetadata { + const promptPath = join( + ROOT, + 'evals', + pair.eval_suite, + pair.eval_id, + 'PROMPT.md' + ); + return parseEvalMarkdown(readFileSync(promptPath, 'utf8'), promptPath) + .metadata; +} + +const EXPERIMENTS_DIR = join(ROOT, 'experiments'); + +let experimentPaths: Promise> | undefined; + +function experimentPathsByName(): Promise> { + experimentPaths ??= discoverExperimentFiles(EXPERIMENTS_DIR).then( + (files) => new Map(files.map((file) => [file.name, file.path])) + ); + return experimentPaths; +} + +/** + * Throws rather than treating a config it can't find as needing no channel, + * which would silently resolve every channel per-sandbox again. + */ +async function loadExperimentConfig( + experiment: string +): Promise { + const path = (await experimentPathsByName()).get(experiment); + if (!path) { + throw new Error( + `no experiment config found for "${experiment}" under ${relative(ROOT, EXPERIMENTS_DIR)}` + ); + } + const mod = await import(pathToFileURL(path).href); + return mod.default as ExperimentConfig; } interface SandboxCommandOptions { @@ -110,6 +219,15 @@ async function runPairs(options: RunnerOptions): Promise { `max ${options.concurrency} at a time` ); + // Resolved once, for only the channels this run's pairs actually need, so + // every job below writes the same pin rather than each sandbox resolving + // its own channel version independently. + const channels = await requiredCliChannels(options.pairs, { + loadEvalMetadata, + loadExperimentConfig, + }); + const pins = await resolveChannelPins(channels); + const results = await runBounded( jobs, options.concurrency, @@ -123,6 +241,7 @@ async function runPairs(options: RunnerOptions): Promise { pair, run, attempt, + pins, }, credentials ), @@ -266,7 +385,7 @@ async function runPairOnce( await sandbox.writeFiles([ { path: '.env', - content: `${agentEnvironment()}\n`, + content: `${agentEnvironment(options.pins)}\n`, }, ]); console.log(`${label} run eval`); @@ -622,11 +741,11 @@ function vercelCredentialsFromEnv(): { }; } -/** Serializes configured provider keys into the repo-root `.env` file. */ -function agentEnvironment(): string { +/** Serializes configured provider keys and CLI channel pins into the sandbox's `.env` file, preferring an explicit pin over the same-named process.env value. */ +export function agentEnvironment(pins: Record = {}): string { const lines: string[] = []; - for (const name of AGENT_ENV_NAMES) { - const value = process.env[name]; + for (const name of FORWARDED_ENV_NAMES) { + const value = pins[name] ?? process.env[name]; if (value) lines.push(`${name}=${value}`); } return lines.join('\n'); diff --git a/packages/core/package.json b/packages/core/package.json index 51ed3f4d..6c41ea2e 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -8,7 +8,8 @@ "exports": { ".": "./src/index.ts", "./eval-metadata": "./src/eval-metadata.ts", - "./eval-markdown": "./src/eval-markdown.ts" + "./eval-markdown": "./src/eval-markdown.ts", + "./json": "./src/json.ts" }, "scripts": { "test": "vitest run" diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index f2550a1c..311fca73 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -582,6 +582,8 @@ export type LocalStackSession = { export type LocalStackRuntime = { id: string; startSession(args: LocalStackSessionArgs): Promise; + /** Channel (e.g. `beta`) this runtime resolves its CLI version against, unset when it's pinned to an exact version. */ + cliChannel?: 'stable' | 'beta'; }; export type ExperimentConfig = { diff --git a/packages/core/src/json.ts b/packages/core/src/json.ts index b8da5d0d..8f7c0369 100644 --- a/packages/core/src/json.ts +++ b/packages/core/src/json.ts @@ -1,4 +1,9 @@ -/** Shared JSON helpers used across the core package. */ +/** + * Shared JSON helpers, imported both from within the core package and, via + * the `@supabase-evals/core/json` subpath, by other packages (e.g. + * `@supabase-evals/sandbox`'s CLI channel resolution) that want them without + * pulling in core's much larger `index.ts`. + */ export function isRecord(value: unknown): value is Record { return typeof value === 'object' && value !== null && !Array.isArray(value); diff --git a/packages/sandbox/package.json b/packages/sandbox/package.json index 41b587a1..9da33f75 100644 --- a/packages/sandbox/package.json +++ b/packages/sandbox/package.json @@ -11,7 +11,7 @@ }, "scripts": { "typecheck": "tsc --noEmit", - "test": "vitest run test/unit.test.ts", + "test": "vitest run test", "test:docker": "SANDBOX_DOCKER_TESTS=1 vitest run test/docker.test.ts" }, "dependencies": { diff --git a/packages/sandbox/src/cli-channel.ts b/packages/sandbox/src/cli-channel.ts new file mode 100644 index 00000000..26167b84 --- /dev/null +++ b/packages/sandbox/src/cli-channel.ts @@ -0,0 +1,242 @@ +/** + * Resolves "latest stable"/"latest beta" Supabase CLI versions from npm's + * dist-tags. npm can point at a still-draft release with no downloadable + * asset, so the resolved version's `.deb` is verified first. + */ + +import { isRecord } from '@supabase-evals/core/json'; + +export type CliChannel = 'stable' | 'beta'; + +const NPM_DIST_TAGS_URL = + 'https://registry.npmjs.org/-/package/supabase/dist-tags'; + +// Makes npm return the abbreviated packument instead of the full one. +const NPM_PACKUMENT_URL = 'https://registry.npmjs.org/supabase'; +const NPM_INSTALL_V1_ACCEPT = 'application/vnd.npm.install-v1+json'; + +// npm's beta dist-tag can carry a prerelease suffix like -rc.1, not just -beta.N. +const VERSION_RE = /^\d+\.\d+\.\d+(-[0-9A-Za-z.]+)?$/; + +// Only the -beta.N shape has a defined ordering the walk-back below can search. +const BETA_SUFFIX_RE = /^\d+\.\d+\.\d+-beta\.\d+$/; + +// Feeds compareBetaVersionsDesc's numeric comparison; matches any version, unlike BETA_SUFFIX_RE. +const BETA_VERSION_RE = /^(\d+)\.(\d+)\.(\d+)-beta\.(\d+)$/; + +// Caps the walk-back so a broken release can't chain into unbounded GitHub requests. +const MAX_BETA_FALLBACK_CANDIDATES = 5; + +const ENV_OVERRIDE: Record = { + stable: 'SUPABASE_CLI_STABLE_VERSION', + beta: 'SUPABASE_CLI_BETA_VERSION', +}; + +const DIST_TAG: Record = { + stable: 'latest', + beta: 'beta', +}; + +const versionCache = new Map>(); + +const CLI_CHANNELS = new Set(['stable', 'beta']); + +export function isCliChannel(value: string): value is CliChannel { + return CLI_CHANNELS.has(value as CliChannel); +} + +/** Must match the download URL installSupabaseCli() uses in supabase.ts. */ +export function cliDebUrl(version: string, arch: 'amd64' | 'arm64'): string { + return `https://github.com/supabase/cli/releases/download/v${version}/supabase_${version}_linux_${arch}.deb`; +} + +/** GitHub release page for a version, used in error messages. */ +function releaseTagUrl(version: string): string { + return `https://github.com/supabase/cli/releases/tag/v${version}`; +} + +/** Only a 404 means the asset is absent; any other non-2xx (e.g. a GitHub 429/5xx) is an error, not a missing-asset signal. */ +async function debAssetHeadOk(url: string): Promise { + const response = await fetch(url, { + method: 'HEAD', + redirect: 'follow', + signal: AbortSignal.timeout(15_000), + }); + if (response.ok) return true; + if (response.status === 404) return false; + throw new Error(`HEAD ${url} -> ${response.status} ${response.statusText}`); +} + +/** HEAD-checks both the amd64 and arm64 `.deb` assets, since the resolver's host architecture need not match the sandbox's. */ +async function debAssetExists(version: string): Promise { + const [amd64, arm64] = await Promise.all([ + debAssetHeadOk(cliDebUrl(version, 'amd64')), + debAssetHeadOk(cliDebUrl(version, 'arm64')), + ]); + return amd64 && arm64; +} + +/** + * Resolves a channel to a concrete CLI version, memoised per channel for the + * process lifetime. A rejection clears the cache entry so the next call + * retries; never falls back to the pinned SUPABASE_CLI_VERSION, since that + * would silently mislabel data. + */ +export async function resolveCliVersion(channel: CliChannel): Promise { + const cached = versionCache.get(channel); + if (cached) return cached; + + const promise = resolveCliVersionUncached(channel); + versionCache.set(channel, promise); + promise.catch(() => { + if (versionCache.get(channel) === promise) versionCache.delete(channel); + }); + return promise; +} + +/** Resolves `value`: a channel tag (`'stable'` | `'beta'`) resolves against npm; an exact version or `undefined` passes through unchanged. */ +export async function resolveCliVersionOption( + value: string | CliChannel | undefined +): Promise { + if (value === undefined) return undefined; + return isCliChannel(value) ? resolveCliVersion(value) : value; +} + +async function resolveCliVersionUncached(channel: CliChannel): Promise { + const envVar = ENV_OVERRIDE[channel]; + const override = process.env[envVar]?.trim().replace(/^v/, ''); + if (override) { + if (!VERSION_RE.test(override)) { + throw new Error( + `${envVar}=${JSON.stringify(override)} is not a valid Supabase CLI version` + ); + } + // An explicit pin is trusted as-is — no asset check, network or otherwise. + return override; + } + + const response = await fetch(NPM_DIST_TAGS_URL, { + signal: AbortSignal.timeout(15_000), + }); + if (!response.ok) { + throw new Error( + `failed to resolve the latest ${channel} Supabase CLI version: ` + + `GET ${NPM_DIST_TAGS_URL} -> ${response.status} ${response.statusText}` + ); + } + const tags = await response.json(); + const distTag = DIST_TAG[channel]; + const version = isRecord(tags) ? tags[distTag] : undefined; + if (typeof version !== 'string' || !VERSION_RE.test(version)) { + throw new Error( + `npm dist-tags for "supabase" did not have a valid "${distTag}" version ` + + `for the ${channel} channel: ${JSON.stringify(version)}` + ); + } + + // A transient failure here throws rather than silently walking back to an + // older beta (or never walking back, for stable). + if (await debAssetExists(version)) return version; + + // npm's dist-tag can point at a version whose GitHub release is still a + // draft with no downloadable .deb. Only beta walks back to an older + // published version; stable is never guessed at. + if (channel === 'stable') { + throw new Error( + `npm's "latest" dist-tag for "supabase" points at ${version}, but its ` + + `release asset is missing (checked amd64 and arm64 .deb assets at ` + + `${releaseTagUrl(version)})` + ); + } + + return resolveFallbackBetaVersion(version); +} + +/** + * Walks back through published `X.Y.Z-beta.N` versions strictly older than + * `unpublishedVersion`, newest first, for one with a downloadable `.deb`. + * Newer candidates are skipped too (likelier to be drafts). Only a 404 + * advances to the next candidate; any other probe error aborts the + * walk-back, since it means GitHub is unhealthy rather than that the + * candidate is absent. + */ +async function resolveFallbackBetaVersion( + unpublishedVersion: string +): Promise { + if (!BETA_SUFFIX_RE.test(unpublishedVersion)) { + throw new Error( + `npm's "beta" dist-tag for "supabase" points at ${unpublishedVersion}, ` + + `whose release asset is missing (checked amd64 and arm64 .deb assets ` + + `at ${releaseTagUrl(unpublishedVersion)}), and its version does not ` + + 'match the X.Y.Z-beta.N shape this fallback can walk back through' + ); + } + + const response = await fetch(NPM_PACKUMENT_URL, { + headers: { Accept: NPM_INSTALL_V1_ACCEPT }, + signal: AbortSignal.timeout(15_000), + }); + if (!response.ok) { + throw new Error( + `failed to look up published beta versions for "supabase": ` + + `GET ${NPM_PACKUMENT_URL} -> ${response.status} ${response.statusText}` + ); + } + const packument = await response.json(); + const versions = isRecord(packument) ? packument.versions : undefined; + if (!isRecord(versions)) { + throw new Error( + `npm packument for "supabase" did not include a "versions" object` + ); + } + + const candidates = Object.keys(versions) + .filter((candidate) => BETA_VERSION_RE.test(candidate)) + .filter( + (candidate) => compareBetaVersionsDesc(candidate, unpublishedVersion) > 0 + ) + .sort(compareBetaVersionsDesc) + .slice(0, MAX_BETA_FALLBACK_CANDIDATES); + + for (const candidate of candidates) { + if (await debAssetExists(candidate)) return candidate; + } + + throw new Error( + `no published beta Supabase CLI release has a downloadable .deb asset; ` + + `checked ${[unpublishedVersion, ...candidates].join(', ')} via ` + + `${NPM_PACKUMENT_URL}` + ); +} + +function parseBetaVersion( + version: string +): [major: number, minor: number, patch: number, beta: number] | undefined { + const match = BETA_VERSION_RE.exec(version); + if (!match) return undefined; + return [ + Number(match[1]), + Number(match[2]), + Number(match[3]), + Number(match[4]), + ]; +} + +/** + * Orders two `X.Y.Z-beta.N` versions newest-first, usable as an + * `Array#sort` comparator; compares components numerically so `beta.10` + * sorts ahead of `beta.9`. Throws if either string isn't parseable. + */ +export function compareBetaVersionsDesc(a: string, b: string): number { + const tupleA = parseBetaVersion(a); + const tupleB = parseBetaVersion(b); + if (!tupleA || !tupleB) { + throw new Error( + `compareBetaVersionsDesc expected "X.Y.Z-beta.N" versions, got ${JSON.stringify(a)} and ${JSON.stringify(b)}` + ); + } + for (let index = 0; index < tupleA.length; index += 1) { + if (tupleA[index] !== tupleB[index]) return tupleB[index] - tupleA[index]; + } + return 0; +} diff --git a/packages/sandbox/src/index.ts b/packages/sandbox/src/index.ts index 793a5e5e..52d68e09 100644 --- a/packages/sandbox/src/index.ts +++ b/packages/sandbox/src/index.ts @@ -1,3 +1,5 @@ +export { resolveCliVersion } from './cli-channel.js'; +export type { CliChannel } from './cli-channel.js'; export { DockerSandbox, dockerCli } from './docker-sandbox.js'; export type { DockerSandboxOptions, diff --git a/packages/sandbox/src/local-stack-runtime.ts b/packages/sandbox/src/local-stack-runtime.ts index 855a3be8..08ba4a71 100644 --- a/packages/sandbox/src/local-stack-runtime.ts +++ b/packages/sandbox/src/local-stack-runtime.ts @@ -15,6 +15,11 @@ import { DockerSandbox } from './docker-sandbox.js'; import { createAgentEnvironment } from './agent-environment.js'; import { ensureEdgeRuntime, teardownSupabaseProject } from './supabase.js'; import { buildSkillsPrompt } from './skills.js'; +import { + isCliChannel, + resolveCliVersionOption, + type CliChannel, +} from './cli-channel.js'; const DEFAULT_BASH_TIMEOUT_SEC = 240; const MAX_BASH_TIMEOUT_SEC = 600; @@ -36,8 +41,13 @@ const STACK_CONFIG_RETRY_MS = 2_000; * profile — out of scope for now. */ export interface LocalStackRuntimeOptions { - /** Supabase CLI version baked into the sandbox image (pinned default). */ - cliVersion?: string; + /** + * Supabase CLI version baked into the sandbox image: an exact version + * (e.g. `2.109.1`) or a channel tag (`'stable'` | `'beta'`) resolved + * against npm's dist-tag and memoised per process. An eval's own + * `cliVersion:` frontmatter pin always wins over this option. + */ + cliVersion?: CliChannel | (string & {}); /** * Supabase MCP feature groups to expose to the agent when the eval links to * a hosted project (`hostedProject: true`). The MCP server runs host-side and @@ -74,6 +84,10 @@ export function localStackRuntime( ): LocalStackRuntime { return { id: 'local-stack', + cliChannel: + options.cliVersion !== undefined && isCliChannel(options.cliVersion) + ? options.cliVersion + : undefined, async startSession({ agent, cliVersion, @@ -86,7 +100,8 @@ export function localStackRuntime( skipCliInstall, }) { const env = await createAgentEnvironment({ - cliVersion: cliVersion ?? options.cliVersion, + cliVersion: + cliVersion ?? (await resolveCliVersionOption(options.cliVersion)), localDir, skills, mounts, diff --git a/packages/sandbox/test/cli-channel.test.ts b/packages/sandbox/test/cli-channel.test.ts new file mode 100644 index 00000000..95a34cbd --- /dev/null +++ b/packages/sandbox/test/cli-channel.test.ts @@ -0,0 +1,611 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { cliDebUrl } from '../src/cli-channel.js'; + +const STABLE_ENV = 'SUPABASE_CLI_STABLE_VERSION'; +const BETA_ENV = 'SUPABASE_CLI_BETA_VERSION'; +const DIST_TAGS_URL = 'https://registry.npmjs.org/-/package/supabase/dist-tags'; +const PACKUMENT_URL = 'https://registry.npmjs.org/supabase'; +const INSTALL_V1_ACCEPT = 'application/vnd.npm.install-v1+json'; + +function jsonResponse( + body: unknown, + init: { ok?: boolean; status?: number; statusText?: string } = {} +) { + return { + ok: init.ok ?? true, + status: init.status ?? 200, + statusText: init.statusText ?? 'OK', + json: async () => body, + }; +} + +function headResponse(ok: boolean, status?: number, statusText?: string) { + return { + ok, + status: status ?? (ok ? 200 : 404), + statusText: statusText ?? (ok ? 'OK' : 'Not Found'), + }; +} + +/** + * Routes a single stubbed `fetch` by method + URL, mirroring the real + * mixture of GET (dist-tags, packument) and HEAD (amd64 + arm64 asset check) + * calls resolveCliVersion makes. + */ +function routedFetchMock(routes: { + distTags?: unknown; + distTagsInit?: { ok?: boolean; status?: number; statusText?: string }; + assetOk?: (version: string, arch: 'amd64' | 'arm64') => boolean; + packument?: unknown; +}) { + return vi.fn(async (url: string, init?: RequestInit) => { + const method = init?.method ?? 'GET'; + if (method === 'HEAD') { + const match = url.match(/supabase_(.+)_linux_(amd64|arm64)\.deb$/); + const [, version, arch] = match ?? []; + return headResponse( + version && arch + ? (routes.assetOk?.(version, arch as 'amd64' | 'arm64') ?? false) + : false + ); + } + if (url === DIST_TAGS_URL) { + return jsonResponse(routes.distTags, routes.distTagsInit); + } + if (url === PACKUMENT_URL) { + return jsonResponse(routes.packument); + } + throw new Error(`unexpected fetch: ${method} ${url}`); + }); +} + +beforeEach(() => { + vi.resetModules(); + delete process.env[STABLE_ENV]; + delete process.env[BETA_ENV]; +}); + +afterEach(() => { + vi.unstubAllGlobals(); + delete process.env[STABLE_ENV]; + delete process.env[BETA_ENV]; +}); + +describe('resolveCliVersion', () => { + it('resolves the stable channel from the "latest" dist-tag when its asset exists', async () => { + const fetchMock = routedFetchMock({ + distTags: { latest: '1.2.3', beta: '1.3.0-beta.1' }, + assetOk: () => true, + }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersion } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersion('stable')).resolves.toBe('1.2.3'); + + const headCalls = fetchMock.mock.calls.filter( + ([, init]) => init?.method === 'HEAD' + ); + expect(headCalls.map(([url]) => url)).toEqual([ + cliDebUrl('1.2.3', 'amd64'), + cliDebUrl('1.2.3', 'arm64'), + ]); + }); + + it('resolves the beta channel from the "beta" dist-tag when its asset exists', async () => { + const fetchMock = routedFetchMock({ + distTags: { latest: '1.2.3', beta: '1.3.0-beta.1' }, + assetOk: () => true, + }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersion } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersion('beta')).resolves.toBe('1.3.0-beta.1'); + }); + + it('prefers the env override over the network and strips a leading v', async () => { + process.env[STABLE_ENV] = 'v9.9.9'; + const fetchMock = vi.fn(); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersion } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersion('stable')).resolves.toBe('9.9.9'); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it('trims surrounding whitespace from the env override', async () => { + process.env[BETA_ENV] = ' 1.4.0-rc.2 '; + vi.stubGlobal('fetch', vi.fn()); + const { resolveCliVersion } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersion('beta')).resolves.toBe('1.4.0-rc.2'); + }); + + it('throws naming the env var when the override is not a valid version', async () => { + process.env[BETA_ENV] = 'not-a-version'; + vi.stubGlobal('fetch', vi.fn()); + const { resolveCliVersion } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersion('beta')).rejects.toThrow( + `${BETA_ENV}="not-a-version" is not a valid Supabase CLI version` + ); + }); + + it('throws with the HTTP status when the registry request fails', async () => { + const fetchMock = routedFetchMock({ + distTags: undefined, + distTagsInit: { + ok: false, + status: 500, + statusText: 'Internal Server Error', + }, + }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersion } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersion('stable')).rejects.toThrow( + `${DIST_TAGS_URL} -> 500 Internal Server Error` + ); + }); + + it('throws when the requested dist-tag is missing from the response', async () => { + const fetchMock = routedFetchMock({ distTags: { beta: '1.0.0-beta.1' } }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersion } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersion('stable')).rejects.toThrow( + /did not have a valid "latest" version for the stable channel/ + ); + }); + + it('throws when the dist-tag value is not a valid version string', async () => { + const fetchMock = routedFetchMock({ + distTags: { latest: 'not-a-version' }, + }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersion } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersion('stable')).rejects.toThrow( + /did not have a valid "latest" version for the stable channel/ + ); + }); + + it('treats an array dist-tags response as invalid rather than a record', async () => { + const fetchMock = routedFetchMock({ distTags: ['not', 'a', 'record'] }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersion } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersion('stable')).rejects.toThrow( + /did not have a valid "latest" version for the stable channel/ + ); + }); + + it('memoises a successful resolution so a second call does not refetch', async () => { + const fetchMock = routedFetchMock({ + distTags: { latest: '1.2.3', beta: '1.3.0-beta.1' }, + assetOk: () => true, + }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersion } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersion('stable')).resolves.toBe('1.2.3'); + await expect(resolveCliVersion('stable')).resolves.toBe('1.2.3'); + + // One GET (dist-tags) + two HEAD (amd64+arm64 asset check) — the second + // call hits the cache. + expect(fetchMock).toHaveBeenCalledTimes(3); + }); + + it('clears the cache entry on rejection so a later call refetches, and keeps the retry memoised', async () => { + let failNextDistTags = true; + const fetchMock = vi.fn(async (url: string, init?: RequestInit) => { + if (init?.method === 'HEAD') return headResponse(true); + if (url === DIST_TAGS_URL) { + if (failNextDistTags) { + failNextDistTags = false; + return jsonResponse(undefined, { + ok: false, + status: 500, + statusText: 'Internal Server Error', + }); + } + return jsonResponse({ latest: '1.2.3', beta: '1.3.0-beta.1' }); + } + throw new Error(`unexpected fetch: ${url}`); + }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersion } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersion('stable')).rejects.toThrow('500'); + await expect(resolveCliVersion('stable')).resolves.toBe('1.2.3'); + // A third call must still hit the cache populated by the successful + // retry — the rejected first promise's cleanup must not evict it. + await expect(resolveCliVersion('stable')).resolves.toBe('1.2.3'); + + expect(fetchMock).toHaveBeenCalledTimes(4); + }); + + it('does not let a beta rejection clear the stable cache entry', async () => { + const fetchMock = vi.fn(async (url: string, init?: RequestInit) => { + if (init?.method === 'HEAD') return headResponse(true); + if (url === DIST_TAGS_URL) { + return jsonResponse({ latest: '1.2.3', beta: 'not-a-version' }); + } + throw new Error(`unexpected fetch: ${url}`); + }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersion } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersion('stable')).resolves.toBe('1.2.3'); + await expect(resolveCliVersion('beta')).rejects.toThrow( + /did not have a valid "beta" version for the beta channel/ + ); + await expect(resolveCliVersion('stable')).resolves.toBe('1.2.3'); + + // Two dist-tags GETs (stable, beta) + two HEAD (stable's amd64+arm64 + // asset check only — beta never gets that far). + expect(fetchMock).toHaveBeenCalledTimes(4); + }); + + it('falls back to the newest published beta.N-1 when the dist-tag asset is a 404', async () => { + const fetchMock = routedFetchMock({ + distTags: { latest: '1.2.3', beta: '2.118.0-beta.52' }, + assetOk: (version) => version !== '2.118.0-beta.52', + packument: { + versions: { + '2.118.0-beta.52': {}, + '2.118.0-beta.51': {}, + '2.118.0-beta.50': {}, + '2.117.0': {}, + }, + }, + }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersion } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersion('beta')).resolves.toBe('2.118.0-beta.51'); + + const packumentCall = fetchMock.mock.calls.find( + ([url]) => url === PACKUMENT_URL + ); + expect(packumentCall?.[1]?.headers).toMatchObject({ + Accept: INSTALL_V1_ACCEPT, + }); + + const headUrls = fetchMock.mock.calls + .filter(([, init]) => init?.method === 'HEAD') + .map(([url]) => url); + expect(headUrls).toEqual([ + cliDebUrl('2.118.0-beta.52', 'amd64'), + cliDebUrl('2.118.0-beta.52', 'arm64'), + cliDebUrl('2.118.0-beta.51', 'amd64'), + cliDebUrl('2.118.0-beta.51', 'arm64'), + ]); + }); + + it("falls back across a minor version boundary to the previous minor's newest beta", async () => { + const fetchMock = routedFetchMock({ + distTags: { latest: '2.118.0', beta: '2.119.0-beta.1' }, + assetOk: (version) => version === '2.118.0-beta.60', + packument: { + versions: { + '2.119.0-beta.1': {}, + '2.118.0-beta.60': {}, + '2.118.0-beta.59': {}, + '2.117.0': {}, + }, + }, + }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersion } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersion('beta')).resolves.toBe('2.118.0-beta.60'); + }); + + it('orders beta.10 ahead of beta.9 during the walk-back (not a string compare)', async () => { + const fetchMock = routedFetchMock({ + distTags: { latest: '1.2.3', beta: '2.118.0-beta.11' }, + // Only the unpublished dist-tag version (11) 404s; both walk-back + // candidates would succeed, so probe order is what decides the result. + assetOk: (version) => version !== '2.118.0-beta.11', + packument: { + versions: { + '2.118.0-beta.11': {}, + '2.118.0-beta.10': {}, + '2.118.0-beta.9': {}, + }, + }, + }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersion } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersion('beta')).resolves.toBe('2.118.0-beta.10'); + + const headUrls = fetchMock.mock.calls + .filter(([, init]) => init?.method === 'HEAD') + .map(([url]) => url); + // A naive string compare would sort "beta.9" ahead of "beta.10" (since + // '9' > '1' character-wise), probing 9 before 10. + expect(headUrls).toEqual([ + cliDebUrl('2.118.0-beta.11', 'amd64'), + cliDebUrl('2.118.0-beta.11', 'arm64'), + cliDebUrl('2.118.0-beta.10', 'amd64'), + cliDebUrl('2.118.0-beta.10', 'arm64'), + ]); + }); + + it('never selects a beta candidate newer than the unpublished dist-tag version', async () => { + const fetchMock = routedFetchMock({ + distTags: { latest: '1.2.3', beta: '2.118.0-beta.50' }, + // Every candidate downloads except the unpublished one — if the newer + // candidate (51) were ever probed, this would resolve to beta.51 + // instead of walking further back to beta.49. + assetOk: (version) => version !== '2.118.0-beta.50', + packument: { + versions: { + '2.118.0-beta.51': {}, + '2.118.0-beta.50': {}, + '2.118.0-beta.49': {}, + }, + }, + }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersion } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersion('beta')).resolves.toBe('2.118.0-beta.49'); + }); + + it('aborts the walk-back and propagates when a candidate probe throws, rather than continuing to the next candidate', async () => { + const fetchMock = vi.fn(async (url: string, init?: RequestInit) => { + if (init?.method === 'HEAD') { + if (url.includes('2.118.0-beta.52')) return headResponse(false); + if (url.includes('2.118.0-beta.51')) { + return headResponse(false, 500, 'Internal Server Error'); + } + if (url.includes('2.118.0-beta.50')) return headResponse(true); + return headResponse(false); + } + if (url === DIST_TAGS_URL) { + return jsonResponse({ latest: '1.2.3', beta: '2.118.0-beta.52' }); + } + if (url === PACKUMENT_URL) { + return jsonResponse({ + versions: { + '2.118.0-beta.52': {}, + '2.118.0-beta.51': {}, + '2.118.0-beta.50': {}, + }, + }); + } + throw new Error(`unexpected fetch: ${url}`); + }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersion } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersion('beta')).rejects.toThrow( + /2\.118\.0-beta\.51.*-> 500 Internal Server Error/ + ); + + const headUrls = fetchMock.mock.calls + .filter(([, init]) => init?.method === 'HEAD') + .map(([url]) => url); + // beta.50 is never probed — the throw on beta.51 aborts the walk-back. + expect(headUrls).toEqual([ + cliDebUrl('2.118.0-beta.52', 'amd64'), + cliDebUrl('2.118.0-beta.52', 'arm64'), + cliDebUrl('2.118.0-beta.51', 'amd64'), + cliDebUrl('2.118.0-beta.51', 'arm64'), + ]); + }); + + it('advances to the second walk-back candidate when the first is a 404', async () => { + const fetchMock = routedFetchMock({ + distTags: { latest: '1.2.3', beta: '2.118.0-beta.52' }, + assetOk: (version) => version === '2.118.0-beta.50', + packument: { + versions: { + '2.118.0-beta.52': {}, + '2.118.0-beta.51': {}, + '2.118.0-beta.50': {}, + }, + }, + }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersion } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersion('beta')).resolves.toBe('2.118.0-beta.50'); + + const headUrls = fetchMock.mock.calls + .filter(([, init]) => init?.method === 'HEAD') + .map(([url]) => url); + expect(headUrls).toEqual([ + cliDebUrl('2.118.0-beta.52', 'amd64'), + cliDebUrl('2.118.0-beta.52', 'arm64'), + cliDebUrl('2.118.0-beta.51', 'amd64'), + cliDebUrl('2.118.0-beta.51', 'arm64'), + cliDebUrl('2.118.0-beta.50', 'amd64'), + cliDebUrl('2.118.0-beta.50', 'arm64'), + ]); + }); + + it('caps the beta walk-back at MAX_BETA_FALLBACK_CANDIDATES, probing newest-first', async () => { + const versions: Record = {}; + for (let n = 50; n <= 59; n += 1) versions[`2.118.0-beta.${n}`] = {}; + const fetchMock = routedFetchMock({ + distTags: { latest: '1.2.3', beta: '2.118.0-beta.60' }, + assetOk: () => false, + packument: { versions }, + }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersion } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersion('beta')).rejects.toThrow(); + + const probedVersions = [ + ...new Set( + fetchMock.mock.calls + .filter(([, init]) => init?.method === 'HEAD') + .map(([url]) => url.match(/supabase_(.+)_linux_/)?.[1]) + ), + ]; + // The unpublished dist-tag version itself is checked first (outside the + // walk-back budget), followed by exactly MAX_BETA_FALLBACK_CANDIDATES (5) + // older published betas, newest-first. + expect(probedVersions).toEqual([ + '2.118.0-beta.60', + '2.118.0-beta.59', + '2.118.0-beta.58', + '2.118.0-beta.57', + '2.118.0-beta.56', + '2.118.0-beta.55', + ]); + }); + + it('throws naming the unpublished versions when no published beta has a downloadable asset', async () => { + const fetchMock = routedFetchMock({ + distTags: { latest: '1.2.3', beta: '2.118.0-beta.52' }, + assetOk: () => false, + packument: { + versions: { + '2.118.0-beta.52': {}, + '2.118.0-beta.51': {}, + }, + }, + }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersion } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersion('beta')).rejects.toThrow( + /2\.118\.0-beta\.52.*2\.118\.0-beta\.51/ + ); + }); + + it('throws instead of guessing when the stable dist-tag asset is a 404', async () => { + const fetchMock = routedFetchMock({ + distTags: { latest: '1.2.3', beta: '1.3.0-beta.1' }, + assetOk: () => false, + }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersion } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersion('stable')).rejects.toThrow( + 'checked amd64 and arm64 .deb assets at ' + + 'https://github.com/supabase/cli/releases/tag/v1.2.3' + ); + }); + + it('throws naming the URL and status when the dist-tag asset HEAD is a GitHub error, without fetching the packument', async () => { + const fetchMock = vi.fn(async (url: string, init?: RequestInit) => { + if (init?.method === 'HEAD') { + return url.includes('_amd64') + ? headResponse(false, 500, 'Internal Server Error') + : headResponse(true); + } + if (url === DIST_TAGS_URL) { + return jsonResponse({ latest: '1.2.3', beta: '2.118.0-beta.52' }); + } + if (url === PACKUMENT_URL) { + throw new Error('packument must not be fetched'); + } + throw new Error(`unexpected fetch: ${url}`); + }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersion } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersion('beta')).rejects.toThrow( + `${cliDebUrl('2.118.0-beta.52', 'amd64')} -> 500 Internal Server Error` + ); + + expect(fetchMock.mock.calls.some(([url]) => url === PACKUMENT_URL)).toBe( + false + ); + }); + + it('does not hit the network at all for an env override', async () => { + process.env[STABLE_ENV] = '9.9.9'; + process.env[BETA_ENV] = '9.9.9-beta.1'; + const fetchMock = vi.fn(); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersion } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersion('stable')).resolves.toBe('9.9.9'); + await expect(resolveCliVersion('beta')).resolves.toBe('9.9.9-beta.1'); + expect(fetchMock).not.toHaveBeenCalled(); + }); +}); + +describe('resolveCliVersionOption', () => { + it('passes an exact version through unchanged, without touching the network', async () => { + const fetchMock = vi.fn(); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersionOption } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersionOption('2.109.1')).resolves.toBe('2.109.1'); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it('passes undefined through unchanged, without touching the network', async () => { + const fetchMock = vi.fn(); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersionOption } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersionOption(undefined)).resolves.toBeUndefined(); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it('resolves a "stable" channel tag against npm', async () => { + const fetchMock = routedFetchMock({ + distTags: { latest: '1.2.3', beta: '1.3.0-beta.1' }, + assetOk: () => true, + }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersionOption } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersionOption('stable')).resolves.toBe('1.2.3'); + }); + + it('resolves a "beta" channel tag against npm', async () => { + const fetchMock = routedFetchMock({ + distTags: { latest: '1.2.3', beta: '1.3.0-beta.1' }, + assetOk: () => true, + }); + vi.stubGlobal('fetch', fetchMock); + const { resolveCliVersionOption } = await import('../src/cli-channel.js'); + + await expect(resolveCliVersionOption('beta')).resolves.toBe('1.3.0-beta.1'); + }); +}); + +describe('compareBetaVersionsDesc', () => { + it('sorts newer beta numbers before older ones within the same minor', async () => { + const { compareBetaVersionsDesc } = await import('../src/cli-channel.js'); + + expect( + compareBetaVersionsDesc('2.118.0-beta.10', '2.118.0-beta.9') + ).toBeLessThan(0); + expect( + compareBetaVersionsDesc('2.118.0-beta.9', '2.118.0-beta.10') + ).toBeGreaterThan(0); + }); + + it('sorts a newer minor before an older minor regardless of beta number', async () => { + const { compareBetaVersionsDesc } = await import('../src/cli-channel.js'); + + expect( + compareBetaVersionsDesc('2.119.0-beta.1', '2.118.0-beta.60') + ).toBeLessThan(0); + }); + + it('treats equal versions as equal', async () => { + const { compareBetaVersionsDesc } = await import('../src/cli-channel.js'); + + expect(compareBetaVersionsDesc('2.118.0-beta.1', '2.118.0-beta.1')).toBe(0); + }); + + it('throws for a non "X.Y.Z-beta.N" version', async () => { + const { compareBetaVersionsDesc } = await import('../src/cli-channel.js'); + + expect(() => + compareBetaVersionsDesc('2.118.0-rc.1', '2.118.0-beta.1') + ).toThrow(); + }); +});