feat(sandbox): resolve npm channel tags for the local-stack CLI version

README.md

@@ -145,6 +145,8 @@ An eval's optional `local/` directory is copied into the sandbox workspace befor
 
 Set `cliVersion: 2.109.1` in an eval's frontmatter when it requires a specific Supabase CLI release. This overrides an experiment's `localStackRuntime({ cliVersion })` setting; otherwise the runtime setting or repository-wide default applies.
 
+An experiment can instead pass `localStackRuntime({ cliVersion: 'stable' })` or `'beta'` to track npm's dist-tag for the `supabase` package rather than an exact version, resolved once at session start. An eval's own `cliVersion:` pin still wins over either form.
+
 Scorers check what the agent produced, never what the harness provisioned: with `projectRunning: true` (the default) the running stack and the seeded `local/` workspace are setup, so score only the deltas the agent made on top; with `projectRunning: false` the agent creates that state itself, so depending on it is fair game.
 
 Test the sandbox plumbing without an agent run (Docker required, not part of `pnpm check`):

apps/framework/scripts/run-vercel-evals.test.ts

@@ -1,4 +1,5 @@
 import { APIError } from '@vercel/sandbox';
+import { resolveCliVersion, type CliChannel } from '@supabase-evals/sandbox';
 import { execFileSync } from 'node:child_process';
 import {
   copyFileSync,
@@ -11,20 +12,253 @@ import {
 } from 'node:fs';
 import { tmpdir } from 'node:os';
 import { dirname, join } from 'node:path';
-import { describe, expect, it } from 'vitest';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
 import {
+  agentEnvironment,
   cleanupSandbox,
   downloadResults,
   finalizeResult,
+  FORWARDED_ENV_NAMES,
   isRetryableSandboxCreateError,
   isTerminalSandboxCreateError,
   packWorkspaceScript,
   parsePairs,
+  requiredCliChannels,
+  resolveChannelPins,
   runBounded,
   tagValue,
   expandJobs,
+  type EvalPair,
 } from './run-vercel-evals.js';
 
+vi.mock('@supabase-evals/sandbox', () => ({ resolveCliVersion: vi.fn() }));
+
+describe('agentEnvironment', () => {
+  const originalValues = new Map<string, string | undefined>();
+
+  beforeEach(() => {
+    for (const name of FORWARDED_ENV_NAMES) {
+      originalValues.set(name, process.env[name]);
+      delete process.env[name];
+    }
+  });
+
+  afterEach(() => {
+    for (const [name, value] of originalValues) {
+      if (value === undefined) delete process.env[name];
+      else process.env[name] = value;
+    }
+  });
+
+  it('forwards every configured name when set', () => {
+    for (const name of FORWARDED_ENV_NAMES) {
+      process.env[name] = `${name}-value`;
+    }
+
+    const lines = agentEnvironment().split('\n');
+    for (const name of FORWARDED_ENV_NAMES) {
+      expect(lines).toContain(`${name}=${name}-value`);
+    }
+  });
+
+  it('omits the CLI channel pins when unset', () => {
+    process.env.ANTHROPIC_API_KEY = 'anthropic-value';
+
+    const env = agentEnvironment();
+    expect(env).toBe('ANTHROPIC_API_KEY=anthropic-value');
+    expect(env).not.toContain('SUPABASE_CLI_STABLE_VERSION');
+    expect(env).not.toContain('SUPABASE_CLI_BETA_VERSION');
+  });
+
+  it('prefers an explicit pin over the same-named process.env value', () => {
+    process.env.SUPABASE_CLI_STABLE_VERSION = 'env-value';
+
+    const env = agentEnvironment({
+      SUPABASE_CLI_STABLE_VERSION: 'pinned-value',
+    });
+
+    expect(env).toContain('SUPABASE_CLI_STABLE_VERSION=pinned-value');
+    expect(env).not.toContain('env-value');
+  });
+
+  it('shares one pin value across multiple .env writes, simulating a two-job fan-out', () => {
+    const pins = {
+      SUPABASE_CLI_STABLE_VERSION: '2.117.0',
+      SUPABASE_CLI_BETA_VERSION: '2.118.0-beta.5',
+    };
+
+    const jobOneEnv = agentEnvironment(pins);
+    const jobTwoEnv = agentEnvironment(pins);
+
+    expect(jobOneEnv).toBe(jobTwoEnv);
+    expect(jobOneEnv).toContain('SUPABASE_CLI_STABLE_VERSION=2.117.0');
+    expect(jobOneEnv).toContain('SUPABASE_CLI_BETA_VERSION=2.118.0-beta.5');
+  });
+});
+
+describe('resolveChannelPins', () => {
+  const STABLE_ENV = 'SUPABASE_CLI_STABLE_VERSION';
+  const BETA_ENV = 'SUPABASE_CLI_BETA_VERSION';
+  const BOTH_CHANNELS = new Set<CliChannel>(['stable', 'beta']);
+  const originalValues = new Map<string, string | undefined>();
+
+  beforeEach(() => {
+    for (const name of [STABLE_ENV, BETA_ENV]) {
+      originalValues.set(name, process.env[name]);
+      delete process.env[name];
+    }
+    vi.mocked(resolveCliVersion).mockReset();
+  });
+
+  afterEach(() => {
+    for (const [name, value] of originalValues) {
+      if (value === undefined) delete process.env[name];
+      else process.env[name] = value;
+    }
+  });
+
+  it('resolves only the requested channels and returns a pin every job can share', async () => {
+    vi.mocked(resolveCliVersion).mockImplementation(async (channel) =>
+      channel === 'stable' ? '2.117.0' : '2.118.0-beta.5'
+    );
+
+    const pins = await resolveChannelPins(BOTH_CHANNELS);
+
+    expect(resolveCliVersion).toHaveBeenCalledTimes(2);
+    expect(resolveCliVersion).toHaveBeenCalledWith('stable');
+    expect(resolveCliVersion).toHaveBeenCalledWith('beta');
+
+    // Two fanned-out jobs writing their own .env from the same pins object
+    // must get the identical value the resolver was called once for.
+    const jobOneEnv = agentEnvironment(pins);
+    const jobTwoEnv = agentEnvironment(pins);
+    expect(jobOneEnv).toBe(jobTwoEnv);
+    expect(jobOneEnv).toContain(`${STABLE_ENV}=2.117.0`);
+    expect(jobOneEnv).toContain(`${BETA_ENV}=2.118.0-beta.5`);
+  });
+
+  it('resolves only stable when that is the only requested channel, never calling resolveCliVersion with beta', async () => {
+    vi.mocked(resolveCliVersion).mockImplementation(async (channel) =>
+      channel === 'stable' ? '2.117.0' : '2.118.0-beta.5'
+    );
+
+    const pins = await resolveChannelPins(new Set<CliChannel>(['stable']));
+
+    expect(pins).toEqual({ [STABLE_ENV]: '2.117.0' });
+    expect(resolveCliVersion).toHaveBeenCalledTimes(1);
+    expect(resolveCliVersion).not.toHaveBeenCalledWith('beta');
+  });
+
+  it('does no network work for an empty channel set', async () => {
+    const pins = await resolveChannelPins(new Set());
+
+    expect(pins).toEqual({});
+    expect(resolveCliVersion).not.toHaveBeenCalled();
+  });
+
+  it('uses an already-set env var verbatim without calling the resolver', async () => {
+    process.env[STABLE_ENV] = '9.9.9';
+    vi.mocked(resolveCliVersion).mockImplementation(
+      async () => '2.118.0-beta.5'
+    );
+
+    const pins = await resolveChannelPins(BOTH_CHANNELS);
+
+    expect(pins[STABLE_ENV]).toBe('9.9.9');
+    expect(resolveCliVersion).toHaveBeenCalledTimes(1);
+    expect(resolveCliVersion).toHaveBeenCalledWith('beta');
+  });
+
+  it('propagates a resolution failure rather than swallowing it', async () => {
+    vi.mocked(resolveCliVersion).mockRejectedValue(
+      new Error('npm unreachable')
+    );
+
+    await expect(
+      resolveChannelPins(new Set<CliChannel>(['stable']))
+    ).rejects.toThrow('npm unreachable');
+  });
+});
+
+describe('requiredCliChannels', () => {
+  const pair = (overrides: Partial<EvalPair> = {}): EvalPair => ({
+    eval_id: 'eval-1',
+    experiment: 'experiment-1',
+    experiment_suite: 'benchmark',
+    eval_suite: 'benchmark',
+    ...overrides,
+  });
+
+  it('resolves only the channel a stable-tagged experiment declares', async () => {
+    const loadExperimentConfig = vi.fn(async () => ({
+      localStack: { cliChannel: 'stable' as const },
+    }));
+
+    const channels = await requiredCliChannels([pair()], {
+      loadEvalMetadata: () => ({ cliVersion: undefined }),
+      loadExperimentConfig,
+    });
+
+    expect(channels).toEqual(new Set(['stable']));
+  });
+
+  it('resolves nothing when no experiment in the pair set declares a channel', async () => {
+    const loadExperimentConfig = vi.fn(async () => ({}));
+
+    const channels = await requiredCliChannels(
+      [pair(), pair({ eval_id: 'eval-2', experiment: 'experiment-2' })],
+      {
+        loadEvalMetadata: () => ({ cliVersion: undefined }),
+        loadExperimentConfig,
+      }
+    );
+
+    expect(channels.size).toBe(0);
+  });
+
+  it("an eval's pinned cliVersion contributes no channel, even when its experiment declares one", async () => {
+    const loadExperimentConfig = vi.fn(async () => ({
+      localStack: { cliChannel: 'beta' as const },
+    }));
+
+    const channels = await requiredCliChannels([pair()], {
+      loadEvalMetadata: () => ({ cliVersion: '2.109.1' }),
+      loadExperimentConfig,
+    });
+
+    expect(channels.size).toBe(0);
+    expect(loadExperimentConfig).not.toHaveBeenCalled();
+  });
+
+  it('unions channels across pairs without resolving an experiment config twice', async () => {
+    const loadExperimentConfig = vi.fn(async () => ({
+      localStack: { cliChannel: 'beta' as const },
+    }));
+
+    const channels = await requiredCliChannels(
+      [pair(), pair({ eval_id: 'eval-2' })],
+      {
+        loadEvalMetadata: () => ({ cliVersion: undefined }),
+        loadExperimentConfig,
+      }
+    );
+
+    expect(channels).toEqual(new Set(['beta']));
+    expect(loadExperimentConfig).toHaveBeenCalledTimes(1);
+  });
+
+  it('throws naming an experiment that cannot be resolved to a config', async () => {
+    await expect(
+      requiredCliChannels([pair({ experiment: 'ghost' })], {
+        loadEvalMetadata: () => ({ cliVersion: undefined }),
+        loadExperimentConfig: async () => {
+          throw new Error('no experiment config found for "ghost"');
+        },
+      })
+    ).rejects.toThrow('no experiment config found for "ghost"');
+  });
+});
+
 describe('Vercel eval controller', () => {
   it('bounds concurrent work and lets independent failures settle', async () => {
     let active = 0;