feat(evals): add resolve-database-003-docker-mode-unavailable CLI eval (CLI-2403)

evals/cli/lib/README.md

@@ -37,6 +37,13 @@ that fails, starts on the staged CLI, then is retried and fails again is flagged
 Global installs and uninstalls are recorded in order whatever their tool call's
 status, since a call's error reflects its last command.
 
+`findSupabaseInvocations` also recognises `pnpm supabase`, `bun x supabase` and
+`yarn supabase`, drops the fd number of a redirect (`2>&1`), and sets
+`experimentalStack` when `SUPABASE_EXPERIMENTAL_STACK` is enabled in the
+invocation's prefix or an earlier `export` in the same command.
+`commandToolCalls` returns the tool calls behind `extractCommands`' entries, so
+an invocation's `commandIndex` maps back to its result.
+
 Invocation attribution (`cli-invocations.ts`) starts from a tool call's `cwd`
 only when the agent parser records one (Codex, OpenCode); a `cd` persisting
 across separate tool calls in a persistent shell (e.g. Claude Code) is not

evals/cli/lib/cli-invocations.test.ts

@@ -3,6 +3,7 @@ import { describe, expect, it } from 'vitest';
 import {
   findSupabaseInvocations,
   invocationDirectory,
+  invocationFlag,
   invocationTargetUnresolved,
   invocationTargets,
   invocationTargetsDir,
@@ -1343,3 +1344,156 @@ describe('exported and home-relative env', () => {
     ).toEqual({ HOME: `${S}/u` });
   });
 });
+
+describe('package runners that run the local binary', () => {
+  const first = (command: string) => findSupabaseInvocations([command])[0];
+
+  it.each([
+    'pnpm supabase start',
+    'bun x supabase start',
+    'npm exec supabase -- start',
+    'yarn supabase start',
+    'pnpm exec supabase start',
+    'npx supabase start',
+  ])('treats %s as a supabase start', (command) => {
+    const invocation = first(command);
+    expect(invocation.argv).toEqual(['supabase', 'start']);
+    expect(invocationVerb(invocation)).toBe('start');
+    expect(isStartInvocation(invocation)).toBe(true);
+    expect(invocation.runner).toBeUndefined();
+  });
+
+  it('records a versioned bun x runner like bunx', () => {
+    expect(first('bun x supabase@2.1.0 stack start')).toMatchObject({
+      argv: ['supabase', 'stack', 'start'],
+      runner: 'bun x supabase@2.1.0',
+    });
+  });
+
+  it('does not treat pnpm or bun installers as supabase runs', () => {
+    expect(
+      findSupabaseInvocations([
+        'pnpm install',
+        'pnpm add supabase',
+        'bun add supabase',
+        'bun x vitest run',
+        'pnpm test',
+      ])
+    ).toEqual([]);
+  });
+
+  it('keeps pnpm add -g as a global install', () => {
+    const [run] = findSupabaseInvocations([
+      'pnpm add -g supabase@2.1.0 && pnpm supabase start',
+    ]);
+    expect(run).toMatchObject({
+      argv: ['supabase', 'start'],
+      runner: 'pnpm add -g supabase@2.1.0',
+    });
+  });
+});
+
+describe('file-descriptor redirects', () => {
+  const argvOf = (command: string) =>
+    findSupabaseInvocations([command])[0].argv;
+
+  it('drops the fd number of a 2>&1 redirect', () => {
+    expect(
+      argvOf('supabase stack start --runtime docker 2>&1 | tail -5')
+    ).toEqual(['supabase', 'stack', 'start', '--runtime', 'docker']);
+  });
+
+  it.each([
+    ['supabase start --runtime native 2>/dev/null', ['--runtime', 'native']],
+    ['supabase start --runtime native 1>out.log', ['--runtime', 'native']],
+    ['supabase start --runtime native 2>> err.log', ['--runtime', 'native']],
+    ['supabase start --runtime native 0< /dev/null', ['--runtime', 'native']],
+  ])('drops the fd number in %s', (command, flags) => {
+    expect(argvOf(command)).toEqual(['supabase', 'start', ...flags]);
+  });
+
+  it('keeps a numeric argument that is not an fd number', () => {
+    expect(argvOf('supabase start --port 54321 2>&1')).toEqual([
+      'supabase',
+      'start',
+      '--port',
+      '54321',
+    ]);
+    expect(argvOf('supabase start --port 2 --debug')).toEqual([
+      'supabase',
+      'start',
+      '--port',
+      '2',
+      '--debug',
+    ]);
+  });
+});
+
+describe('experimentalStack', () => {
+  const first = (command: string) => findSupabaseInvocations([command])[0];
+
+  it.each([
+    'SUPABASE_EXPERIMENTAL_STACK=1 supabase start',
+    'SUPABASE_EXPERIMENTAL_STACK=true supabase start',
+    'env SUPABASE_EXPERIMENTAL_STACK=1 supabase start',
+    'env -u FOO SUPABASE_EXPERIMENTAL_STACK=1 supabase start',
+    'export SUPABASE_EXPERIMENTAL_STACK=1 && supabase start',
+    'export SUPABASE_EXPERIMENTAL_STACK=1; cd app; supabase start',
+    '(export SUPABASE_EXPERIMENTAL_STACK=1; supabase start)',
+    'SUPABASE_EXPERIMENTAL_STACK=1 pnpm supabase start',
+  ])('is set for %s', (command) => {
+    expect(first(command).experimentalStack).toBe(true);
+  });
+
+  it.each([
+    'supabase start',
+    'SUPABASE_EXPERIMENTAL_STACK=0 supabase start',
+    'SUPABASE_EXPERIMENTAL_STACK=false supabase start',
+    'SUPABASE_EXPERIMENTAL_STACK= supabase start',
+    'export SUPABASE_EXPERIMENTAL_STACK=0 && supabase start',
+    'SUPABASE_EXPERIMENTAL_STACK=1; supabase start',
+    'export SUPABASE_EXPERIMENTAL_STACK=1 && env -u SUPABASE_EXPERIMENTAL_STACK supabase start',
+    'export SUPABASE_EXPERIMENTAL_STACK=1 && env -i supabase start',
+    'export SUPABASE_EXPERIMENTAL_STACK=1 && SUPABASE_EXPERIMENTAL_STACK=0 supabase start',
+  ])('is unset for %s', (command) => {
+    expect(first(command).experimentalStack).toBeUndefined();
+  });
+
+  it('does not leak an export out of a closed subshell or into the next command', () => {
+    const invocations = findSupabaseInvocations([
+      '(export SUPABASE_EXPERIMENTAL_STACK=1; supabase start); supabase start',
+      'supabase start',
+    ]);
+    expect(
+      invocations.map(({ experimentalStack }) => experimentalStack)
+    ).toEqual([true, undefined, undefined]);
+  });
+
+  it('is independent of the home variables', () => {
+    expect(
+      first('SUPABASE_EXPERIMENTAL_STACK=1 SUPABASE_HOME=/a supabase start')
+    ).toMatchObject({
+      env: { SUPABASE_HOME: '/a' },
+      experimentalStack: true,
+    });
+    expect(
+      first('SUPABASE_EXPERIMENTAL_STACK=1 supabase start').env
+    ).toBeUndefined();
+  });
+});
+
+describe('invocationFlag', () => {
+  const flag = (command: string, name = '--runtime') =>
+    invocationFlag(findSupabaseInvocations([command])[0], name);
+
+  it('reads both spellings of a flag value', () => {
+    expect(flag('supabase stack start --runtime native')).toBe('native');
+    expect(flag('supabase stack start --runtime=native')).toBe('native');
+    expect(flag('supabase stack start --runtime "docker"')).toBe('docker');
+  });
+
+  it('is undefined when the flag is absent', () => {
+    expect(flag('supabase stack start')).toBeUndefined();
+    expect(flag('supabase stack start --stack a', '--runtime')).toBeUndefined();
+  });
+});

evals/cli/lib/cli-invocations.ts

@@ -30,6 +30,8 @@ export type SupabaseInvocation = {
   globalInstall?: 'active' | 'removed';
   /** The tool call's completion time (epoch ms), present only when the agent parser records it. */
   at?: number;
+  /** Set when `SUPABASE_EXPERIMENTAL_STACK` was enabled (anything but empty, `0` or `false`) in the invocation's own prefix or by an earlier `export` in the same command. */
+  experimentalStack?: true;
 };
 
 export type CommandEntry = {
@@ -72,6 +74,14 @@ const CHDIR_WORDS = new Set(['cd', 'pushd']);
 const WORKDIR_VARIABLE = 'SUPABASE_WORKDIR';
 const WORKDIR_ASSIGNMENT = `${WORKDIR_VARIABLE}=`;
 const ENV_VARIABLES = ['SUPABASE_HOME', 'HOME', 'TMPDIR'] as const;
+const EXPERIMENTAL_STACK_VARIABLE = 'SUPABASE_EXPERIMENTAL_STACK';
+const TRACKED_VARIABLES = [
+  ...ENV_VARIABLES,
+  EXPERIMENTAL_STACK_VARIABLE,
+] as const;
+const FD_REDIRECT_OPERATORS = new Set(['>', '>>', '>&', '<', '<&', '<<<']);
+const FD_NUMBER_RE = /^\d+$/;
+const FALSY_FLAG_VALUES = new Set(['', '0', 'false']);
 const PWD_PREFIX_RE = /^\$\{?PWD\}?(?=\/|$)/;
 const HOME_PREFIX_RE = /^(?:\$\{?HOME\}?|~)(?=\/|$)/;
 const VERSIONED_SUPABASE_RE = /^supabase@.+/;
@@ -133,18 +143,27 @@ function parseKeepingVariables(text: string): ParseEntry[] | undefined {
   }
 }
 
-/** Executed words up to the first operator (redirection, subshell close); globs keep their pattern. */
+/** Executed words up to the first operator (redirection, subshell close); globs keep their pattern, and a file-descriptor number directly before a redirect (`2>&1`) is not a word. */
 function words(tokens: readonly ParseEntry[]): string[] {
   const out: string[] = [];
   for (const token of tokens) {
     if (typeof token === 'string') out.push(token);
     else if ('op' in token && token.op === 'glob') out.push(token.pattern);
-    else break;
+    else {
+      if (
+        'op' in token &&
+        FD_REDIRECT_OPERATORS.has(token.op) &&
+        FD_NUMBER_RE.test(out[out.length - 1] ?? '')
+      ) {
+        out.pop();
+      }
+      break;
+    }
   }
   return out;
 }
 
-type RawEnv = Partial<Record<(typeof ENV_VARIABLES)[number], string>>;
+type RawEnv = Partial<Record<(typeof TRACKED_VARIABLES)[number], string>>;
 
 type Stripped = {
   argv: string[];
@@ -206,6 +225,7 @@ function stripPrefixes(
     const runnerSkip = PACKAGE_RUNNERS.has(word)
       ? 1
       : (word === 'npm' && NPM_EXEC_VERBS.has(next)) ||
+          (word === 'bun' && next === 'x') ||
           (word === 'pnpm' && next === 'dlx') ||
           (word === 'yarn' && next === 'dlx')
         ? 2
@@ -232,9 +252,7 @@ function stripPrefixes(
       }
     } else if (ENV_ASSIGNMENT_RE.test(word)) {
       const name = word.slice(0, word.indexOf('='));
-      if ((ENV_VARIABLES as readonly string[]).includes(name)) {
-        rawEnv[name as keyof RawEnv] = word.slice(name.length + 1);
-      }
+      if (isEnvName(name)) rawEnv[name] = word.slice(name.length + 1);
       i++;
     } else if (PASSTHROUGH_WORDS.has(word)) {
       i++;
@@ -249,6 +267,9 @@ function stripPrefixes(
     } else if (word === 'pnpm' && next === 'exec') {
       runnerStart = undefined;
       i += 2;
+    } else if (word === 'pnpm' && isSupabaseBinary(next)) {
+      runnerStart = undefined;
+      i++;
     } else if (word === 'yarn' && next !== 'global') {
       runnerStart = undefined;
       i++;
@@ -346,7 +367,7 @@ function executedArgv(segment: string, seedEnv: RawEnv): Stripped {
   return stripPrefixes(tokens === undefined ? [] : words(tokens), seedEnv);
 }
 
-type EnvName = (typeof ENV_VARIABLES)[number];
+type EnvName = (typeof TRACKED_VARIABLES)[number];
 
 /** Shell state a command carries from segment to segment, restored when its subshell closes. */
 type ShellState = {
@@ -356,7 +377,7 @@ type ShellState = {
 };
 
 function isEnvName(name: string): name is EnvName {
-  return (ENV_VARIABLES as readonly string[]).includes(name);
+  return (TRACKED_VARIABLES as readonly string[]).includes(name);
 }
 
 function exportedEnv(state: ShellState): RawEnv {
@@ -477,7 +498,7 @@ export function findSupabaseInvocations(
       } else if (argv[0] === 'export') {
         state = assignVariables(state, exportAssignments(argv.slice(1)), true);
       } else if (argv.length === 0) {
-        const assigned = ENV_VARIABLES.filter(
+        const assigned = TRACKED_VARIABLES.filter(
           (name) => rawEnv[name] !== seed[name]
         ).map((name) => [name, rawEnv[name]] as const);
         state = assignVariables(state, assigned, false);
@@ -488,6 +509,7 @@ export function findSupabaseInvocations(
         const env = resolveEnv(rawEnv, state.cwd, dir, state.vars.HOME);
         const effectiveRunner = runner ?? globalRunner;
         const fromGlobal = runner === undefined && globalRunner !== undefined;
+        const experimental = rawEnv[EXPERIMENTAL_STACK_VARIABLE];
         const invocation: SupabaseInvocation = {
           commandIndex,
           argv: ['supabase', ...argv.slice(1)],
@@ -497,6 +519,10 @@ export function findSupabaseInvocations(
           ...(effectiveRunner === undefined ? {} : { runner: effectiveRunner }),
           ...(fromGlobal ? { globalInstall: 'active' as const } : {}),
           ...(at === undefined ? {} : { at }),
+          ...(experimental === undefined ||
+          FALSY_FLAG_VALUES.has(experimental.toLowerCase())
+            ? {}
+            : { experimentalStack: true as const }),
         };
         invocations.push(invocation);
         if (fromGlobal) globalInvocations.push(invocation);
@@ -539,6 +565,14 @@ function flagValue(argv: readonly string[], flag: string): string | undefined {
   return undefined;
 }
 
+/** The value of `--flag value` or `--flag=value` in the invocation's argv. */
+export function invocationFlag(
+  inv: SupabaseInvocation,
+  flag: string
+): string | undefined {
+  return flagValue(inv.argv, flag);
+}
+
 /** The subcommand an invocation ran, e.g. `stop` or `stack destroy`. */
 export function invocationVerb(inv: SupabaseInvocation): string | undefined {
   const [first, second] = positionals(inv.argv);