diff --git a/README.md b/README.md index 5d2a3807..b1ee8e53 100644 --- a/README.md +++ b/README.md @@ -147,6 +147,8 @@ Set `cliVersion: 2.109.1` in an eval's frontmatter when it requires a specific S An experiment can instead pass `localStackRuntime({ cliVersion: 'stable' })` or `'beta'` to track npm's dist-tag for the `supabase` package rather than an exact version, resolved once at session start. An eval's own `cliVersion:` pin still wins over either form. +An experiment can pass `localStackRuntime({ docker: 'no-daemon' })` or `'absent'` to stage a sandbox where the Docker daemon is unreachable or the `docker` binary is missing entirely, instead of the default `'available'`. `needsDocker` defaults to `true`; set it `false` in an eval's frontmatter when the scenario can run, and is meaningful, without a Docker daemon (e.g. starting the stack is the agent's own job) — that's what lets a Docker-less experiment pick the eval up. + Scorers check what the agent produced, never what the harness provisioned: with `projectRunning: true` (the default) the running stack and the seeded `local/` workspace are setup, so score only the deltas the agent made on top; with `projectRunning: false` the agent creates that state itself, so depending on it is fair game. Test the sandbox plumbing without an agent run (Docker required, not part of `pnpm check`): diff --git a/packages/core/src/eval-metadata.ts b/packages/core/src/eval-metadata.ts index 328a9d60..fa699c88 100644 --- a/packages/core/src/eval-metadata.ts +++ b/packages/core/src/eval-metadata.ts @@ -154,6 +154,14 @@ export type EvalMetadata = { * with `projectRunning: false`. */ skipCliInstall?: boolean; + /** + * Whether this scenario needs a working Docker daemon (sandbox evals + * only). Defaults to true. Set false when the scenario can run, and is + * meaningful, in a sandbox without Docker (e.g. starting the stack is the + * agent's own job) — Docker-less experiments filter on this to decide + * which evals they can run. + */ + needsDocker?: boolean; }; export type ParsedEvalMarkdown = { @@ -175,6 +183,7 @@ export const evalMetadataSchema = z.object({ hostedProject: z.union([z.boolean(), z.stringbool()]).optional(), skills: z.array(z.string().min(1)).optional(), skipCliInstall: z.union([z.boolean(), z.stringbool()]).optional(), + needsDocker: z.union([z.boolean(), z.stringbool()]).optional(), }); // Collapse a YAML scalar into a comparable token: trim, lowercase, and fold @@ -253,6 +262,7 @@ export const evalFrontmatterSchema = z.preprocess((raw) => { ? toIdentifierList(data.skills) : undefined, skipCliInstall: data.skipCliInstall, + needsDocker: data.needsDocker, }; }, evalMetadataSchema); diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 73bd5f77..f07a0d96 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -303,6 +303,20 @@ export interface LocalStackStatus { anonKey: string; } +/** + * Parsed contents of the environment marker a local-stack session writes, + * recording its staged CLI version and Docker availability — for scorers to + * report, never to gate pass/fail. + */ +export interface LocalStackEnvironmentMarker { + runtime: 'local-stack'; + /** The channel (`stable`/`beta`) the session's CLI version resolved from, when the runtime named one. */ + channel?: 'stable' | 'beta'; + cliVersion: string; + docker: 'available' | 'no-daemon' | 'absent'; + sessionStartedMs: number; +} + /** * Scoring surface for local-stack evals. Everything runs inside the Docker * sandbox the agent worked in, against the local Supabase stack it (or the @@ -342,6 +356,13 @@ export interface LocalStackScoringContext { * doesn't cover, or to assert on where a key ended up. */ stackStatus: () => Promise; + /** + * The session's environment marker, read directly from the sandbox as + * root so nothing on the agent's `PATH` can forge it, and never through + * `resolveSandboxPath`/`exec`. Undefined when the session recorded none. + * For scorers to report, never to gate pass/fail. + */ + environmentMarker: () => Promise; /** * The mocked hosted project's ref, when the eval links to platform-lite * (`hostedProject: true`). Undefined for purely-local evals. diff --git a/packages/sandbox/src/cli-channel.ts b/packages/sandbox/src/cli-channel.ts index 26167b84..1ed1f5ac 100644 --- a/packages/sandbox/src/cli-channel.ts +++ b/packages/sandbox/src/cli-channel.ts @@ -41,6 +41,7 @@ const versionCache = new Map>(); const CLI_CHANNELS = new Set(['stable', 'beta']); +/** Whether `value` names a channel (`'stable'` | `'beta'`) rather than an exact version. */ export function isCliChannel(value: string): value is CliChannel { return CLI_CHANNELS.has(value as CliChannel); } diff --git a/packages/sandbox/src/docker-sandbox.ts b/packages/sandbox/src/docker-sandbox.ts index 360cd67b..77a222f5 100644 --- a/packages/sandbox/src/docker-sandbox.ts +++ b/packages/sandbox/src/docker-sandbox.ts @@ -87,6 +87,13 @@ export interface DockerSandboxOptions { * tools must execute — e.g. a local MCP server build. */ mounts?: readonly SandboxMount[]; + /** + * Bind-mount the host Docker socket into the container. Defaults to true. + * Set false for sandboxes that must be provably Docker-less — the socket + * is never exposed to the container at all, rather than merely hidden or + * blocked from inside it. + */ + mountDockerSocket?: boolean; } export interface RunCommandOptions { @@ -100,6 +107,7 @@ export class DockerSandbox { private network: string | undefined; private image: string; private mounts: readonly SandboxMount[]; + private mountDockerSocket: boolean; readonly workdir: string; /** * Env vars injected into every `runShell` (non-root) command — both the @@ -113,6 +121,7 @@ export class DockerSandbox { this.network = options.network; this.image = options.image ?? DEFAULT_IMAGE; this.mounts = options.mounts ?? []; + this.mountDockerSocket = options.mountDockerSocket !== false; this.workdir = `${WORKSPACE_BASE}-${randomUUID().slice(0, 8)}`; } @@ -141,8 +150,9 @@ export class DockerSandbox { '--rm', '--label', `${SANDBOX_CONTAINER_LABEL}=1`, - '--volume', - '/var/run/docker.sock:/var/run/docker.sock', + ...(this.mountDockerSocket + ? ['--volume', '/var/run/docker.sock:/var/run/docker.sock'] + : []), '--volume', `${this.workdir}:${this.workdir}`, // Caller-requested host mounts (e.g. a local MCP server build the @@ -331,7 +341,8 @@ export class DockerSandbox { * with an explicit mode — e.g. installing a small CLI shim onto PATH. Content * is staged on the host and `docker cp`'d in, so it arrives as data with no * shell quoting, and the copy runs as the engine, so it can write system dirs - * regardless of the container user. The chmod then sets the mode as root. + * regardless of the container user. `docker cp` preserves host ownership, so + * one checked root command then both chowns the file to root and sets its mode. */ async writeRootFile( containerPath: string, @@ -347,14 +358,37 @@ export class DockerSandbox { } finally { rmSync(staging, { recursive: true, force: true }); } - const chmod = await this.runShellAsRoot( - `chmod ${mode} ${shellQuote(containerPath)}` + const secure = await this.runShellAsRoot( + `chown root:root ${shellQuote(containerPath)} && chmod ${mode} ${shellQuote(containerPath)}` ); - if (!chmod.ok) { - throw new Error(`failed to chmod ${containerPath}: ${chmod.stderr}`); + if (!secure.ok) { + throw new Error(`failed to secure ${containerPath}: ${secure.stderr}`); } } + /** + * Read an absolute container path as root, in exec form with an absolute + * binary rather than a shell, so nothing on the agent's `PATH` (including + * `SANDBOX_PATH`'s writable first entry) can shadow the read. + */ + async readRootFile(containerPath: string): Promise { + this.assertRunning(); + const result = await dockerCli([ + 'exec', + '--user', + 'root', + this.containerId!, + '/bin/cat', + containerPath, + ]); + if (!result.ok) { + throw new Error( + `failed to read ${containerPath} as root: ${result.stderr}` + ); + } + return result.stdout; + } + private async execCommand( command: string, options: { env?: Record; user: string; timeoutMs?: number } diff --git a/packages/sandbox/src/index.ts b/packages/sandbox/src/index.ts index 52d68e09..8cfc4fb7 100644 --- a/packages/sandbox/src/index.ts +++ b/packages/sandbox/src/index.ts @@ -19,13 +19,20 @@ export { } from './supabase.js'; export type { SetupSupabaseSandboxOptions } from './supabase.js'; export { + buildDockerDaemonShimScript, buildLocalStackScoringContext, buildLocalStackTools, + buildSupabaseShimScript, buildToolSurfaceAddendum, + LOCAL_STACK_MARKER_PATH, localStackRuntime, toAgentSandbox, } from './local-stack-runtime.js'; -export type { LocalStackRuntimeOptions } from './local-stack-runtime.js'; +export type { + DockerState, + LocalStackEnvironmentMarker, + LocalStackRuntimeOptions, +} from './local-stack-runtime.js'; export { SKILLS_CLI_VERSION, SKILLS_INSTALL_AGENTS, diff --git a/packages/sandbox/src/local-stack-runtime.ts b/packages/sandbox/src/local-stack-runtime.ts index 08ba4a71..b3564ace 100644 --- a/packages/sandbox/src/local-stack-runtime.ts +++ b/packages/sandbox/src/local-stack-runtime.ts @@ -6,20 +6,30 @@ import { type AgentHarnessId, type AgentSandbox, type HostedLink, + type LocalStackEnvironmentMarker, type LocalStackRuntime, type LocalStackScoringContext, type LocalStackStatus, type McpServerConfig, } from '@supabase-evals/core'; +import { isRecord } from '@supabase-evals/core/json'; import { DockerSandbox } from './docker-sandbox.js'; import { createAgentEnvironment } from './agent-environment.js'; -import { ensureEdgeRuntime, teardownSupabaseProject } from './supabase.js'; -import { buildSkillsPrompt } from './skills.js'; +import { + computeExcludedServices, + ensureEdgeRuntime, + ensureSupabaseSandboxImage, + installSupabaseCli, + SUPABASE_CLI_VERSION, + teardownSupabaseProject, +} from './supabase.js'; +import { buildSkillsPrompt, installSkills } from './skills.js'; import { isCliChannel, resolveCliVersionOption, type CliChannel, } from './cli-channel.js'; +import type { SupabaseService } from './types.js'; const DEFAULT_BASH_TIMEOUT_SEC = 240; const MAX_BASH_TIMEOUT_SEC = 600; @@ -65,8 +75,44 @@ export interface LocalStackRuntimeOptions { * sandbox; pass `{}` to disable MCP altogether. */ mcpServers?: Record; + /** + * Docker availability to stage in the sandbox (default `'available'`). + * `'no-daemon'` leaves `docker --version` working but unreachable; + * `'absent'` removes the binary entirely. Both require + * `projectRunning: false` and no hosted project link, since the harness + * cannot pre-start a stack or link a hosted project without Docker. + */ + docker?: DockerState; } +/** + * Docker's availability inside a local-stack sandbox session. See + * {@link LocalStackRuntimeOptions.docker}. + */ +export type DockerState = 'available' | 'no-daemon' | 'absent'; + +/** + * Path of the marker each session writes recording its staged environment, + * for scorers to report, never to gate pass/fail. Agent-writable on the + * `'available'` path (metrics-only); root-owned and read-only (0444) on the + * Docker-less paths, so the agent can't fake the environment it's graded in. + */ +export const LOCAL_STACK_MARKER_PATH = '/tmp/supabase-eval-runtime.json'; + +// The shared scoring-context shape (LocalStackScoringContext.environmentMarker) +// lives in core; re-exported here so existing imports of this type from +// @supabase-evals/sandbox keep working. +export type { LocalStackEnvironmentMarker } from '@supabase-evals/core'; + +// Shadow the real `docker`/`supabase` binaries from the first entry on the +// sandbox PATH (see SANDBOX_PATH in docker-sandbox.ts). +const SUPABASE_SHIM_PATH = '/usr/local/sbin/supabase'; +const DOCKER_SHIM_PATH = '/usr/local/sbin/docker'; + +// Port 1 is never bound (the CLI's own e2e suite reserves it for this too); +// not 2375, which Docker Desktop can legitimately expose. +const UNREACHABLE_DOCKER_HOST = 'tcp://127.0.0.1:1'; + /** * Supabase MCP feature groups exposed by default: `docs` only. The sandbox has * no web tools, so `search_docs` is the one capability the agent otherwise @@ -83,7 +129,7 @@ export function localStackRuntime( options: LocalStackRuntimeOptions = {} ): LocalStackRuntime { return { - id: 'local-stack', + id: buildRuntimeId(options), cliChannel: options.cliVersion !== undefined && isCliChannel(options.cliVersion) ? options.cliVersion @@ -99,53 +145,353 @@ export function localStackRuntime( mounts, skipCliInstall, }) { - const env = await createAgentEnvironment({ - cliVersion: - cliVersion ?? (await resolveCliVersionOption(options.cliVersion)), - localDir, - skills, + // Stamped before setup so it's comparable with the scorer's PID-1 fallback. + const sessionStartedMs = Date.now(); + const docker = options.docker ?? 'available'; + // Only an unpinned eval inherits a channel; an eval's own pin always wins. + const channel = + cliVersion === undefined && + options.cliVersion !== undefined && + isCliChannel(options.cliVersion) + ? options.cliVersion + : undefined; + const version = + cliVersion ?? + (await resolveCliVersionOption(options.cliVersion)) ?? + SUPABASE_CLI_VERSION; + + if (docker === 'available') { + const env = await createAgentEnvironment({ + cliVersion: version, + localDir, + skills, + mounts, + localStack: { + includeServices, + projectRunning, + hosted: hosted + ? { + port: hosted.port, + pgPort: hosted.pgPort, + ref: hosted.ref, + accessToken: hosted.accessToken, + } + : undefined, + skipCliInstall, + }, + }); + const sandbox = env.sandbox; + + const mcpServers = await resolveMcpServers(options, hosted); + + const session = { + tools: buildLocalStackTools(sandbox, docker), + sandbox: toAgentSandbox(sandbox), + mcpServers, + promptAddendum: [ + buildToolSurfaceAddendum(agent, { skipCliInstall, docker }), + buildSkillsPrompt(agent, env.skills), + ] + .filter(Boolean) + .join('\n\n'), + scoringContext: buildLocalStackScoringContext(sandbox, hosted), + ensureReady: () => ensureEdgeRuntime(sandbox, includeServices), + exportWorkspace: (hostDir: string) => + sandbox.copyToHost(sandbox.workdir, hostDir), + close: async () => { + await teardownSupabaseProject(sandbox); + await env.close(); + }, + }; + + // Metrics-only and best-effort here; the Docker-less path's marker + // is root-owned and fails hard instead. + try { + await writeLocalStackMarkerViaExec( + (command) => session.scoringContext.exec(command), + buildLocalStackMarker(docker, version, sessionStartedMs, channel) + ); + } catch (err) { + console.warn( + `[local-stack] could not write ${LOCAL_STACK_MARKER_PATH}: ${ + err instanceof Error ? err.message : String(err) + }` + ); + } + return session; + } + + // Docker-less staging: the harness cannot pre-start a stack or link a + // hosted project without Docker. + if (projectRunning !== false) { + throw new Error( + 'docker-less sandbox evals must set `projectRunning: false`; the harness cannot pre-start a stack without Docker' + ); + } + if (hosted) { + throw new Error( + 'docker-less sandbox evals cannot link a hosted project — set hostedProject: false' + ); + } + + const image = await ensureSupabaseSandboxImage(); + const sandbox = await DockerSandbox.create({ + image, + network: 'host', mounts, - localStack: { - includeServices, - projectRunning, - hosted: hosted - ? { - port: hosted.port, - pgPort: hosted.pgPort, - ref: hosted.ref, - accessToken: hosted.accessToken, - } - : undefined, - skipCliInstall, - }, + mountDockerSocket: false, }); - const sandbox = env.sandbox; - - const mcpServers = await resolveMcpServers(options, hosted); - - return { - tools: buildLocalStackTools(sandbox), - sandbox: toAgentSandbox(sandbox), - mcpServers, - promptAddendum: [ - buildToolSurfaceAddendum(agent, { skipCliInstall }), - buildSkillsPrompt(agent, env.skills), - ] - .filter(Boolean) - .join('\n\n'), - scoringContext: buildLocalStackScoringContext(sandbox, hosted), - ensureReady: () => ensureEdgeRuntime(sandbox, includeServices), - exportWorkspace: (hostDir: string) => - sandbox.copyToHost(sandbox.workdir, hostDir), - close: async () => { - await teardownSupabaseProject(sandbox); - await env.close(); - }, - }; + + try { + if (!skipCliInstall) { + await installSupabaseCli(sandbox, version); + } + + // No socket-group grant (unlike setupSupabaseSandbox): CI's sandbox + // setup ends with `chmod 666 /var/run/docker.sock`, which would make + // any permission-based restriction a no-op — DOCKER_HOST below is + // what actually blocks access. Left set for `absent` too since it's + // harmless and guards against future socket exposure. + sandbox.extraEnv = { + ...sandbox.extraEnv, + DOCKER_HOST: UNREACHABLE_DOCKER_HOST, + }; + + if (!skipCliInstall) { + await installSupabaseShim(sandbox, includeServices); + } + + // Captured before removal: no-daemon's shim echoes this for `docker + // --version` so the CLI's #6563 runtime probe still picks Docker. + let dockerVersion = ''; + if (docker === 'no-daemon') { + dockerVersion = ( + await sandbox.runShellAsRoot('docker --version') + ).stdout.trim(); + if (!dockerVersion) { + throw new Error( + 'failed to capture `docker --version` before removing the real binary' + ); + } + } + + // Assert the postcondition instead of trusting `rm -f`, which always exits 0. + const removal = await sandbox.runShellAsRoot( + 'for b in docker dockerd docker-proxy; do p="$(command -v "$b" 2>/dev/null)" && rm -f "$p"; done; ! command -v docker >/dev/null 2>&1' + ); + if (!removal.ok) { + throw new Error( + `docker is still on PATH after removal: ${removal.stderr || removal.stdout}` + ); + } + + // The root shell's PATH differs from the agent's SANDBOX_PATH, so + // also assert the binary is gone from the PATH the agent runs under. + const pathCheck = await sandbox.runShell( + '! command -v docker >/dev/null 2>&1' + ); + if (!pathCheck.ok) { + throw new Error( + `docker is still on the agent's PATH after removal: ${pathCheck.stderr || pathCheck.stdout}` + ); + } + + if (docker === 'no-daemon') { + await installDockerDaemonShim(sandbox, dockerVersion); + } + + if (localDir) { + await sandbox.copyToContainer(localDir, sandbox.workdir); + } + const installedSkills = await installSkills(sandbox, skills ?? []); + + // No bind mount means the socket must not exist at all — stronger + // than checking reachability from inside the container. + const socketAbsent = await sandbox.runShellAsRoot( + 'test ! -e /var/run/docker.sock' + ); + if (!socketAbsent.ok) { + throw new Error( + 'the Docker socket unexpectedly exists in a Docker-less sandbox' + ); + } + + // Root-owned and read-only so the agent cannot rewrite it to fake + // the environment it's being evaluated in. + await sandbox.writeRootFile( + LOCAL_STACK_MARKER_PATH, + JSON.stringify( + buildLocalStackMarker(docker, version, sessionStartedMs, channel) + ), + '0444' + ); + + const dockerlessMcpServers = await resolveMcpServers( + options, + undefined + ); + + return { + tools: buildLocalStackTools(sandbox, docker), + sandbox: toAgentSandbox(sandbox), + // Same wiring as the `available` path; `hosted` is always + // undefined here (guarded above), so this falls back to `docs`. + mcpServers: dockerlessMcpServers, + promptAddendum: [ + buildToolSurfaceAddendum(agent, { skipCliInstall, docker }), + buildSkillsPrompt(agent, installedSkills), + ] + .filter(Boolean) + .join('\n\n'), + scoringContext: buildLocalStackScoringContext(sandbox), + exportWorkspace: (hostDir: string) => + sandbox.copyToHost(sandbox.workdir, hostDir), + // Nothing to restore without Docker; the `available` path above + // delegates to its own session's ensureReady. + ensureReady: async () => {}, + // No teardownSupabaseProject: nothing can have started without Docker. + close: () => sandbox.stop(), + }; + } catch (err) { + await sandbox.stop(); + throw err; + } }, }; } +// Log-line id (see run-eval.ts's PLAN line): default options read as +// 'local-stack', otherwise the non-default bits are appended, e.g. +// 'local-stack-beta-absent'. +function buildRuntimeId(options: LocalStackRuntimeOptions): string { + const bits: string[] = []; + if (options.cliVersion !== undefined) bits.push(options.cliVersion); + const docker = options.docker ?? 'available'; + if (docker !== 'available') bits.push(docker); + return bits.length > 0 ? `local-stack-${bits.join('-')}` : 'local-stack'; +} + +function buildLocalStackMarker( + docker: DockerState, + cliVersion: string, + sessionStartedMs: number, + channel?: CliChannel +): LocalStackEnvironmentMarker { + return { + runtime: 'local-stack', + channel, + cliVersion, + docker, + sessionStartedMs, + }; +} + +async function writeLocalStackMarkerViaExec( + exec: ( + command: string + ) => Promise<{ ok: boolean; stdout: string; stderr: string }>, + marker: LocalStackEnvironmentMarker +): Promise { + // base64 transport sidesteps quoting the JSON payload through the shell. + const encoded = Buffer.from(JSON.stringify(marker), 'utf-8').toString( + 'base64' + ); + const result = await exec( + `echo ${encoded} | base64 -d > ${LOCAL_STACK_MARKER_PATH}` + ); + if (!result.ok) { + throw new Error( + `failed to write the local-stack environment marker: ${result.stderr || result.stdout}` + ); + } +} + +async function installSupabaseShim( + sandbox: DockerSandbox, + includeServices: readonly string[] | undefined +): Promise { + const real = ( + await sandbox.runShellAsRoot('command -v supabase') + ).stdout.trim(); + if (!real || real === SUPABASE_SHIM_PATH) { + throw new Error( + `could not resolve the real supabase binary before installing the CLI shim (got ${JSON.stringify(real)})` + ); + } + const excluded = computeExcludedServices(includeServices); + await sandbox.writeRootFile( + SUPABASE_SHIM_PATH, + buildSupabaseShimScript(real, excluded), + '0755' + ); +} + +/** + * Shim that shadows `supabase` on PATH in a Docker-less sandbox. Here the + * agent, not the harness, runs `supabase start`, so this is the only route + * by which the eval's `includeServices` still gets applied (`-x `). + */ +export function buildSupabaseShimScript( + realBin: string, + excluded: readonly SupabaseService[] +): string { + const lines = [ + '#!/bin/bash', + `export DOCKER_HOST=${UNREACHABLE_DOCKER_HOST}`, + `REAL=${shellQuote(realBin)}`, + ]; + if (excluded.length > 0) { + lines.push( + `if [ "$1" = "start" ]; then shift; exec "$REAL" start "$@" -x ${excluded.join(',')}; fi` + ); + } + lines.push('exec "$REAL" "$@"'); + return lines.join('\n'); +} + +async function installDockerDaemonShim( + sandbox: DockerSandbox, + dockerVersion: string +): Promise { + await sandbox.writeRootFile( + DOCKER_SHIM_PATH, + buildDockerDaemonShimScript(dockerVersion), + '0755' + ); +} + +export function buildDockerDaemonShimScript(dockerVersion: string): string { + return [ + '#!/bin/bash', + 'case "$1" in', + // --version must keep working so the CLI's runtime probe + // (supabase/cli#6563) still chooses Docker as its runtime. + ` --version|-v) echo ${shellQuote(dockerVersion)}; exit 0 ;;`, + 'esac', + `echo "Cannot connect to the Docker daemon at ${UNREACHABLE_DOCKER_HOST}. Is the docker daemon running?" >&2`, + 'exit 1', + ].join('\n'); +} + +/** + * Names the workspace's binaries accurately for the resolved Docker state, + * so the Docker-less paths don't claim a binary or daemon the agent doesn't + * have. + */ +function describeDockerTools(docker: DockerState): string { + switch (docker) { + case 'available': + return 'docker, psql, git, and curl are installed in the workspace'; + case 'no-daemon': + return ( + 'psql, git, and curl are installed in the workspace; docker is ' + + 'installed but there is no daemon for it to connect to' + ); + case 'absent': + return 'psql, git, and curl are installed in the workspace; docker is not installed'; + } +} + /** * Describes the session's tool surface: the binaries installed in the workspace * and the in-process `bash`/`files_*` tools from `buildLocalStackTools`. @@ -156,18 +502,24 @@ export function localStackRuntime( */ export function buildToolSurfaceAddendum( agent: AgentHarnessId, - options: { skipCliInstall?: boolean } = {} + options: { skipCliInstall?: boolean; docker?: DockerState } = {} ): string { if (agent !== 'ai-sdk') return ''; + const docker = options.docker ?? 'available'; let addendum = - 'docker, psql, git, and curl are installed in the workspace. ' + + `${describeDockerTools(docker)}. ` + 'Use the bash tool to run commands (the working directory is always the workspace root) ' + 'and the files tools to inspect and modify files.'; if (!options.skipCliInstall) { addendum = 'The Supabase CLI (`supabase`), ' + addendum; - addendum += - ' Services started with `supabase start` are reachable on their default 127.0.0.1 ports.'; + // Without a reachable daemon, `supabase start` cannot bring services up + // at all, so the reachability claim would be false on both Docker-less + // states. + if (docker === 'available') { + addendum += + ' Services started with `supabase start` are reachable on their default 127.0.0.1 ports.'; + } } return addendum; } @@ -181,7 +533,7 @@ export function buildToolSurfaceAddendum( * hosted project there's no platform to talk to, so fall back to the * platform-independent docs server (`search_docs`). */ -async function resolveMcpServers( +export async function resolveMcpServers( options: LocalStackRuntimeOptions, hosted?: HostedLink ): Promise> { @@ -217,13 +569,16 @@ export function toAgentSandbox(sandbox: DockerSandbox): AgentSandbox { }; } -export function buildLocalStackTools(sandbox: DockerSandbox): ToolSet { +export function buildLocalStackTools( + sandbox: DockerSandbox, + docker: DockerState = 'available' +): ToolSet { return { bash: tool({ description: 'Run a bash command in the eval workspace (Linux). The working directory ' + 'is always the workspace root; `cd` does not persist between calls. The ' + - 'Supabase CLI (`supabase`), docker, psql, git, and curl are installed.', + `Supabase CLI (\`supabase\`), ${describeDockerTools(docker)}.`, inputSchema: jsonSchema({ type: 'object', properties: { @@ -350,6 +705,26 @@ export function buildLocalStackScoringContext( ): LocalStackScoringContext { let stackConfig: LocalStackStatus | undefined; let dbUrl: string | undefined; + let environmentMarker: LocalStackEnvironmentMarker | undefined; + let environmentMarkerRead = false; + + // Read as root, in exec form (docker-sandbox.ts's readRootFile), so + // neither `resolveSandboxPath` nor the agent's `PATH` is in the path of a + // scorer reading the environment it's being graded in. + const discoverEnvironmentMarker = async () => { + if (environmentMarkerRead) return environmentMarker; + environmentMarkerRead = true; + try { + const raw = await sandbox.readRootFile(LOCAL_STACK_MARKER_PATH); + const parsed = JSON.parse(raw); + environmentMarker = isLocalStackEnvironmentMarker(parsed) + ? parsed + : undefined; + } catch { + environmentMarker = undefined; + } + return environmentMarker; + }; // Read the DB connection string from the running stack rather than assuming // the default 127.0.0.1:54322 — same derive-from-`supabase status` approach as @@ -440,6 +815,7 @@ export function buildLocalStackScoringContext( return { rows: text ? JSON.parse(text) : [] }; }, stackStatus: () => discoverStackConfig(), + environmentMarker: () => discoverEnvironmentMarker(), getClient: async () => { const { apiUrl, publishableKey } = await discoverStackConfig(); return createClient(apiUrl, publishableKey, { @@ -453,6 +829,23 @@ export function buildLocalStackScoringContext( }; } +function isLocalStackEnvironmentMarker( + value: unknown +): value is LocalStackEnvironmentMarker { + return ( + isRecord(value) && + value.runtime === 'local-stack' && + typeof value.cliVersion === 'string' && + (value.docker === 'available' || + value.docker === 'no-daemon' || + value.docker === 'absent') && + typeof value.sessionStartedMs === 'number' && + (value.channel === undefined || + value.channel === 'stable' || + value.channel === 'beta') + ); +} + function readString( config: Record | undefined, key: string diff --git a/packages/sandbox/test/docker.test.ts b/packages/sandbox/test/docker.test.ts index bba78d49..846a54e2 100644 --- a/packages/sandbox/test/docker.test.ts +++ b/packages/sandbox/test/docker.test.ts @@ -9,7 +9,10 @@ import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterAll, describe, expect, it } from 'vitest'; -import { buildLocalStackScoringContext } from '../src/local-stack-runtime.js'; +import { + buildLocalStackScoringContext, + localStackRuntime, +} from '../src/local-stack-runtime.js'; import { DockerSandbox } from '../src/docker-sandbox.js'; import { ensureSupabaseSandboxImage, @@ -237,5 +240,72 @@ describe.runIf(process.env.SANDBOX_DOCKER_TESTS)( } } ); + + it( + 'a no-daemon sandbox root-owns the marker and both shims, read-only to the agent', + { timeout: TEST_TIMEOUT_MS }, + async () => { + // Docker-less staging guards both projectRunning: false and no hosted + // link, since the harness cannot pre-start a stack or link a hosted + // project without Docker. + const session = await localStackRuntime({ + docker: 'no-daemon', + }).startSession({ agent: 'ai-sdk', projectRunning: false }); + try { + const stat = await session.scoringContext.exec( + 'stat -c "%u:%g %a" /tmp/supabase-eval-runtime.json /usr/local/sbin/supabase /usr/local/sbin/docker' + ); + expect(stat.ok, stat.stderr).toBe(true); + expect(stat.stdout.trim().split('\n')).toEqual([ + '0:0 444', + '0:0 755', + '0:0 755', + ]); + + const write = await session.scoringContext.exec( + 'echo forged > /tmp/supabase-eval-runtime.json' + ); + expect(write.ok).toBe(false); + expect(write.stderr).toMatch(/permission denied/i); + } finally { + await session.close(); + } + } + ); + + it( + 'environmentMarker ignores a cat shadowed onto the front of the agent PATH', + { timeout: TEST_TIMEOUT_MS }, + async () => { + const session = await localStackRuntime({ + docker: 'no-daemon', + }).startSession({ agent: 'ai-sdk', projectRunning: false }); + try { + // /home/node/.npm-global/bin is first on SANDBOX_PATH and writable + // by the agent, so a shell-based read could be fed forged JSON. + const forged = + '{"runtime":"local-stack","cliVersion":"0.0.0",' + + '"docker":"available","sessionStartedMs":0}'; + const shim = ['#!/bin/sh', `echo '${forged}'`, ''].join('\n'); + const encoded = Buffer.from(shim, 'utf-8').toString('base64'); + + const shadow = await session.scoringContext.exec( + 'mkdir -p /home/node/.npm-global/bin && ' + + `echo ${encoded} | base64 -d > /home/node/.npm-global/bin/cat && ` + + 'chmod 755 /home/node/.npm-global/bin/cat' + ); + expect(shadow.ok, shadow.stderr).toBe(true); + + const shadowed = await session.scoringContext.exec('cat /dev/null'); + expect(shadowed.stdout).toContain('"docker":"available"'); + + const marker = await session.scoringContext.environmentMarker(); + expect(marker?.docker).toBe('no-daemon'); + expect(marker?.cliVersion).not.toBe('0.0.0'); + } finally { + await session.close(); + } + } + ); } ); diff --git a/packages/sandbox/test/local-stack-docker.test.ts b/packages/sandbox/test/local-stack-docker.test.ts new file mode 100644 index 00000000..727f34b0 --- /dev/null +++ b/packages/sandbox/test/local-stack-docker.test.ts @@ -0,0 +1,246 @@ +import { describe, expect, it, vi } from 'vitest'; +import { + buildDockerDaemonShimScript, + buildLocalStackScoringContext, + buildSupabaseShimScript, + localStackRuntime, + resolveMcpServers, +} from '../src/local-stack-runtime.js'; +import type { DockerSandbox } from '../src/docker-sandbox.js'; +import type { SupabaseService } from '../src/types.js'; + +describe('localStackRuntime id', () => { + it('ids as `local-stack` with no options', () => { + expect(localStackRuntime().id).toBe('local-stack'); + }); + + it('ids as `local-stack` when docker is explicitly the default', () => { + expect(localStackRuntime({ docker: 'available' }).id).toBe('local-stack'); + }); + + it('appends the docker state when it is non-default', () => { + expect(localStackRuntime({ docker: 'no-daemon' }).id).toBe( + 'local-stack-no-daemon' + ); + expect(localStackRuntime({ docker: 'absent' }).id).toBe( + 'local-stack-absent' + ); + }); + + it('appends the cliVersion option when set', () => { + expect(localStackRuntime({ cliVersion: 'beta' }).id).toBe( + 'local-stack-beta' + ); + expect(localStackRuntime({ cliVersion: '2.109.1' }).id).toBe( + 'local-stack-2.109.1' + ); + }); + + it('combines a non-default cliVersion and docker state', () => { + expect(localStackRuntime({ cliVersion: 'beta', docker: 'absent' }).id).toBe( + 'local-stack-beta-absent' + ); + }); +}); + +describe('buildSupabaseShimScript', () => { + it('emits DOCKER_HOST, the -x start branch for excluded services, and a passthrough exec', () => { + const excluded: SupabaseService[] = ['gotrue', 'kong']; + expect(buildSupabaseShimScript('/usr/bin/supabase', excluded)).toEqual( + [ + '#!/bin/bash', + 'export DOCKER_HOST=tcp://127.0.0.1:1', + "REAL='/usr/bin/supabase'", + 'if [ "$1" = "start" ]; then shift; exec "$REAL" start "$@" -x gotrue,kong; fi', + 'exec "$REAL" "$@"', + ].join('\n') + ); + }); + + it('omits the start branch entirely when no services are excluded', () => { + expect( + buildSupabaseShimScript('/usr/bin/supabase', []) + ).toMatchInlineSnapshot(` + "#!/bin/bash + export DOCKER_HOST=tcp://127.0.0.1:1 + REAL='/usr/bin/supabase' + exec "$REAL" "$@"" + `); + }); + + it('always ends with exec "$REAL" "$@" regardless of exclusions', () => { + const excluded: SupabaseService[] = ['gotrue', 'kong']; + const script = buildSupabaseShimScript('/usr/bin/supabase', excluded); + expect(script.endsWith('exec "$REAL" "$@"')).toBe(true); + }); + + it('single-quotes a realBin containing a single quote safely for the shell', () => { + const script = buildSupabaseShimScript("/usr/local/it's/bin/supabase", []); + expect(script).toContain(`REAL='/usr/local/it'\\''s/bin/supabase'`); + }); + + it('single-quotes a realBin containing $, backtick, and " safely for the shell', () => { + const realBin = '/usr/local/$(whoami)/`id`/"bin"/supabase'; + const script = buildSupabaseShimScript(realBin, []); + const realLine = script + .split('\n') + .find((line) => line.startsWith('REAL=')); + expect(realLine).toBe('REAL=\'/usr/local/$(whoami)/`id`/"bin"/supabase\''); + }); +}); + +describe('buildDockerDaemonShimScript', () => { + const dockerVersion = 'Docker version 20.10.24+dfsg1, build 297e128'; + + it('echoes the captured version string verbatim for --version/-v and exits 0, otherwise fails as unreachable', () => { + expect(buildDockerDaemonShimScript(dockerVersion)).toMatchInlineSnapshot(` + "#!/bin/bash + case "$1" in + --version|-v) echo 'Docker version 20.10.24+dfsg1, build 297e128'; exit 0 ;; + esac + echo "Cannot connect to the Docker daemon at tcp://127.0.0.1:1. Is the docker daemon running?" >&2 + exit 1" + `); + }); + + it('single-quotes a version string containing a single quote safely for the shell', () => { + const script = buildDockerDaemonShimScript( + "Docker version 'weird', build x" + ); + const versionLine = script + .split('\n') + .find((line) => line.includes('--version|-v')); + expect(versionLine).toBe( + ` --version|-v) echo 'Docker version '\\''weird'\\'', build x'; exit 0 ;;` + ); + }); +}); + +describe('resolveMcpServers', () => { + // The Docker-less session path resolves its MCP map through this same + // helper rather than hardcoding a docs-only server, so an experiment's + // explicit wiring survives in a sandbox with no Docker. `hosted` is always + // undefined there, which is the case pinned here. + it('returns an explicit mcpServers map untouched', async () => { + const explicit = { + custom: { command: 'node', args: ['server.js'] }, + }; + await expect( + resolveMcpServers({ mcpServers: explicit }, undefined) + ).resolves.toBe(explicit); + }); + + it('honours an explicit empty map, disabling MCP entirely', async () => { + await expect( + resolveMcpServers({ mcpServers: {} }, undefined) + ).resolves.toEqual({}); + }); + + it('falls back to a single docs-only supabase server', async () => { + const servers = await resolveMcpServers({}, undefined); + expect(Object.keys(servers)).toEqual(['supabase']); + }); +}); + +describe('buildLocalStackScoringContext environmentMarker', () => { + /** A DockerSandbox stub whose readRootFile returns/throws as configured. */ + function fakeSandbox(readRootFile: DockerSandbox['readRootFile']) { + return { + workdir: '/tmp/sandbox-x', + readRootFile, + } as unknown as DockerSandbox; + } + + for (const docker of ['available', 'no-daemon', 'absent'] as const) { + it(`parses a well-formed marker for docker: ${docker}`, async () => { + const marker = { + runtime: 'local-stack', + cliVersion: '2.109.1', + docker, + sessionStartedMs: 1_700_000_000_000, + }; + const readRootFile = vi.fn().mockResolvedValue(JSON.stringify(marker)); + const ctx = buildLocalStackScoringContext(fakeSandbox(readRootFile)); + await expect(ctx.environmentMarker()).resolves.toEqual(marker); + }); + } + + it('includes an optional channel when present', async () => { + const marker = { + runtime: 'local-stack', + channel: 'beta', + cliVersion: '2.109.1', + docker: 'available', + sessionStartedMs: 1_700_000_000_000, + }; + const readRootFile = vi.fn().mockResolvedValue(JSON.stringify(marker)); + const ctx = buildLocalStackScoringContext(fakeSandbox(readRootFile)); + await expect(ctx.environmentMarker()).resolves.toEqual(marker); + }); + + it('reads the marker as root, not through exec or resolveSandboxPath', async () => { + const readRootFile = vi.fn().mockResolvedValue( + JSON.stringify({ + runtime: 'local-stack', + cliVersion: '2.109.1', + docker: 'available', + sessionStartedMs: 0, + }) + ); + const ctx = buildLocalStackScoringContext(fakeSandbox(readRootFile)); + await ctx.environmentMarker(); + expect(readRootFile).toHaveBeenCalledWith( + '/tmp/supabase-eval-runtime.json' + ); + }); + + it('is undefined when no marker was written', async () => { + const readRootFile = vi + .fn() + .mockRejectedValue(new Error('cat: No such file or directory')); + const ctx = buildLocalStackScoringContext(fakeSandbox(readRootFile)); + await expect(ctx.environmentMarker()).resolves.toBeUndefined(); + }); + + it('is undefined for malformed JSON rather than throwing', async () => { + const readRootFile = vi.fn().mockResolvedValue('not json'); + const ctx = buildLocalStackScoringContext(fakeSandbox(readRootFile)); + await expect(ctx.environmentMarker()).resolves.toBeUndefined(); + }); + + it('is undefined for a well-formed but wrong-shaped payload', async () => { + const readRootFile = vi + .fn() + .mockResolvedValue(JSON.stringify({ runtime: 'not-local-stack' })); + const ctx = buildLocalStackScoringContext(fakeSandbox(readRootFile)); + await expect(ctx.environmentMarker()).resolves.toBeUndefined(); + }); + + it('rejects an invalid docker value instead of casting it through', async () => { + const readRootFile = vi.fn().mockResolvedValue( + JSON.stringify({ + runtime: 'local-stack', + cliVersion: '2.109.1', + docker: 'forged', + sessionStartedMs: 0, + }) + ); + const ctx = buildLocalStackScoringContext(fakeSandbox(readRootFile)); + await expect(ctx.environmentMarker()).resolves.toBeUndefined(); + }); + + it('caches the marker instead of re-reading on every call', async () => { + const readRootFile = vi.fn().mockResolvedValue( + JSON.stringify({ + runtime: 'local-stack', + cliVersion: '2.109.1', + docker: 'available', + sessionStartedMs: 0, + }) + ); + const ctx = buildLocalStackScoringContext(fakeSandbox(readRootFile)); + await ctx.environmentMarker(); + await ctx.environmentMarker(); + expect(readRootFile).toHaveBeenCalledTimes(1); + }); +}); diff --git a/packages/sandbox/test/unit.test.ts b/packages/sandbox/test/unit.test.ts index 9f15aa00..643579ef 100644 --- a/packages/sandbox/test/unit.test.ts +++ b/packages/sandbox/test/unit.test.ts @@ -2,6 +2,7 @@ import { readFileSync } from 'node:fs'; import { parseEvalMarkdown } from '@supabase-evals/core/eval-markdown'; import { afterEach, describe, expect, it, vi } from 'vitest'; import { + buildLocalStackTools, buildToolSurfaceAddendum, resolveSandboxPath, truncateOutput, @@ -244,6 +245,75 @@ describe('buildToolSurfaceAddendum', () => { ); } }); + + it('defaults to the docker-available text when docker is unspecified', () => { + expect(buildToolSurfaceAddendum('ai-sdk')).toBe( + buildToolSurfaceAddendum('ai-sdk', { docker: 'available' }) + ); + }); + + it('claims docker is installed and reachable only when docker is available', () => { + const addendum = buildToolSurfaceAddendum('ai-sdk', { + docker: 'available', + }); + expect(addendum).toContain('docker, psql, git, and curl are installed'); + expect(addendum).toContain( + 'Services started with `supabase start` are reachable on their default 127.0.0.1 ports.' + ); + }); + + it('describes docker as installed but daemon-less, with no reachability claim, for no-daemon', () => { + const addendum = buildToolSurfaceAddendum('ai-sdk', { + docker: 'no-daemon', + }); + expect(addendum).toContain('psql, git, and curl are installed'); + expect(addendum).toContain( + 'docker is installed but there is no daemon for it to connect to' + ); + expect(addendum).not.toContain('docker, psql, git, and curl'); + expect(addendum).not.toContain( + 'reachable on their default 127.0.0.1 ports' + ); + }); + + it('describes docker as not installed, with no reachability claim, for absent', () => { + const addendum = buildToolSurfaceAddendum('ai-sdk', { docker: 'absent' }); + expect(addendum).toContain('psql, git, and curl are installed'); + expect(addendum).toContain('docker is not installed'); + expect(addendum).not.toContain('docker, psql, git, and curl'); + expect(addendum).not.toContain( + 'reachable on their default 127.0.0.1 ports' + ); + }); +}); + +describe('buildLocalStackTools bash description', () => { + function bashDescription(docker: 'available' | 'no-daemon' | 'absent') { + const tools = buildLocalStackTools({} as DockerSandbox, docker); + return (tools.bash as { description?: string }).description ?? ''; + } + + it('names docker as installed for the default (available) state', () => { + expect(bashDescription('available')).toContain( + 'docker, psql, git, and curl are installed' + ); + }); + + it('names docker as installed but daemon-less for no-daemon', () => { + const description = bashDescription('no-daemon'); + expect(description).toContain('psql, git, and curl are installed'); + expect(description).toContain( + 'docker is installed but there is no daemon for it to connect to' + ); + expect(description).not.toContain('docker, psql, git, and curl'); + }); + + it('names docker as not installed for absent', () => { + const description = bashDescription('absent'); + expect(description).toContain('psql, git, and curl are installed'); + expect(description).toContain('docker is not installed'); + expect(description).not.toContain('docker, psql, git, and curl'); + }); }); describe('SKILLS_CLI_VERSION', () => { @@ -643,6 +713,43 @@ describe('skipCliInstall frontmatter', () => { }); }); +describe('needsDocker frontmatter', () => { + const buildMarkdown = (extra: string) => + [ + '---', + 'stage: build', + 'interface: cli', + 'product: [database]', + 'topic: [sdk]', + extra, + '---', + 'body', + ].join('\n'); + + it('accepts a real boolean true and false', () => { + expect( + parseEvalMarkdown(buildMarkdown('needsDocker: true')).metadata.needsDocker + ).toBe(true); + expect( + parseEvalMarkdown(buildMarkdown('needsDocker: false')).metadata + .needsDocker + ).toBe(false); + }); + + it('accepts a quoted string form via z.stringbool()', () => { + expect( + parseEvalMarkdown(buildMarkdown('needsDocker: "false"')).metadata + .needsDocker + ).toBe(false); + }); + + it('defaults to undefined when omitted', () => { + expect( + parseEvalMarkdown(buildMarkdown('')).metadata.needsDocker + ).toBeUndefined(); + }); +}); + describe('resolveSandboxPath', () => { it('accepts and normalizes relative paths', () => { expect(resolveSandboxPath('a/b.txt')).toBe('a/b.txt');