feat(sandbox): stage Docker-less local stacks via a docker option

README.md

@@ -147,6 +147,8 @@ Set `cliVersion: 2.109.1` in an eval's frontmatter when it requires a specific S
 
 An experiment can instead pass `localStackRuntime({ cliVersion: 'stable' })` or `'beta'` to track npm's dist-tag for the `supabase` package rather than an exact version, resolved once at session start. An eval's own `cliVersion:` pin still wins over either form.
 
+An experiment can pass `localStackRuntime({ docker: 'no-daemon' })` or `'absent'` to stage a sandbox where the Docker daemon is unreachable or the `docker` binary is missing entirely, instead of the default `'available'`. `needsDocker` defaults to `true`; set it `false` in an eval's frontmatter when the scenario can run, and is meaningful, without a Docker daemon (e.g. starting the stack is the agent's own job) — that's what lets a Docker-less experiment pick the eval up.
+
 Scorers check what the agent produced, never what the harness provisioned: with `projectRunning: true` (the default) the running stack and the seeded `local/` workspace are setup, so score only the deltas the agent made on top; with `projectRunning: false` the agent creates that state itself, so depending on it is fair game.
 
 Test the sandbox plumbing without an agent run (Docker required, not part of `pnpm check`):

packages/core/src/eval-metadata.ts

@@ -154,6 +154,14 @@ export type EvalMetadata = {
    * with `projectRunning: false`.
    */
   skipCliInstall?: boolean;
+  /**
+   * Whether this scenario needs a working Docker daemon (sandbox evals
+   * only). Defaults to true. Set false when the scenario can run, and is
+   * meaningful, in a sandbox without Docker (e.g. starting the stack is the
+   * agent's own job) — Docker-less experiments filter on this to decide
+   * which evals they can run.
+   */
+  needsDocker?: boolean;
 };
 
 export type ParsedEvalMarkdown = {
@@ -175,6 +183,7 @@ export const evalMetadataSchema = z.object({
   hostedProject: z.union([z.boolean(), z.stringbool()]).optional(),
   skills: z.array(z.string().min(1)).optional(),
   skipCliInstall: z.union([z.boolean(), z.stringbool()]).optional(),
+  needsDocker: z.union([z.boolean(), z.stringbool()]).optional(),
 });
 
 // Collapse a YAML scalar into a comparable token: trim, lowercase, and fold
@@ -253,6 +262,7 @@ export const evalFrontmatterSchema = z.preprocess((raw) => {
       ? toIdentifierList(data.skills)
       : undefined,
     skipCliInstall: data.skipCliInstall,
+    needsDocker: data.needsDocker,
   };
 }, evalMetadataSchema);
 

packages/core/src/index.ts

@@ -303,6 +303,20 @@ export interface LocalStackStatus {
   anonKey: string;
 }
 
+/**
+ * Parsed contents of the environment marker a local-stack session writes,
+ * recording its staged CLI version and Docker availability — for scorers to
+ * report, never to gate pass/fail.
+ */
+export interface LocalStackEnvironmentMarker {
+  runtime: 'local-stack';
+  /** The channel (`stable`/`beta`) the session's CLI version resolved from, when the runtime named one. */
+  channel?: 'stable' | 'beta';
+  cliVersion: string;
+  docker: 'available' | 'no-daemon' | 'absent';
+  sessionStartedMs: number;
+}
+
 /**
  * Scoring surface for local-stack evals. Everything runs inside the Docker
  * sandbox the agent worked in, against the local Supabase stack it (or the
@@ -342,6 +356,13 @@ export interface LocalStackScoringContext {
    * doesn't cover, or to assert on where a key ended up.
    */
   stackStatus: () => Promise<LocalStackStatus>;
+  /**
+   * The session's environment marker, read directly from the sandbox as
+   * root so nothing on the agent's `PATH` can forge it, and never through
+   * `resolveSandboxPath`/`exec`. Undefined when the session recorded none.
+   * For scorers to report, never to gate pass/fail.
+   */
+  environmentMarker: () => Promise<LocalStackEnvironmentMarker | undefined>;
   /**
    * The mocked hosted project's ref, when the eval links to platform-lite
    * (`hostedProject: true`). Undefined for purely-local evals.

packages/sandbox/src/cli-channel.ts

@@ -41,6 +41,7 @@ const versionCache = new Map<CliChannel, Promise<string>>();
 
 const CLI_CHANNELS = new Set<CliChannel>(['stable', 'beta']);
 
+/** Whether `value` names a channel (`'stable'` | `'beta'`) rather than an exact version. */
 export function isCliChannel(value: string): value is CliChannel {
   return CLI_CHANNELS.has(value as CliChannel);
 }

packages/sandbox/src/docker-sandbox.ts

@@ -87,6 +87,13 @@ export interface DockerSandboxOptions {
    * tools must execute — e.g. a local MCP server build.
    */
   mounts?: readonly SandboxMount[];
+  /**
+   * Bind-mount the host Docker socket into the container. Defaults to true.
+   * Set false for sandboxes that must be provably Docker-less — the socket
+   * is never exposed to the container at all, rather than merely hidden or
+   * blocked from inside it.
+   */
+  mountDockerSocket?: boolean;
 }
 
 export interface RunCommandOptions {
@@ -100,6 +107,7 @@ export class DockerSandbox {
   private network: string | undefined;
   private image: string;
   private mounts: readonly SandboxMount[];
+  private mountDockerSocket: boolean;
   readonly workdir: string;
   /**
    * Env vars injected into every `runShell` (non-root) command — both the
@@ -113,6 +121,7 @@ export class DockerSandbox {
     this.network = options.network;
     this.image = options.image ?? DEFAULT_IMAGE;
     this.mounts = options.mounts ?? [];
+    this.mountDockerSocket = options.mountDockerSocket !== false;
     this.workdir = `${WORKSPACE_BASE}-${randomUUID().slice(0, 8)}`;
   }
 
@@ -141,8 +150,9 @@ export class DockerSandbox {
         '--rm',
         '--label',
         `${SANDBOX_CONTAINER_LABEL}=1`,
-        '--volume',
-        '/var/run/docker.sock:/var/run/docker.sock',
+        ...(this.mountDockerSocket
+          ? ['--volume', '/var/run/docker.sock:/var/run/docker.sock']
+          : []),
         '--volume',
         `${this.workdir}:${this.workdir}`,
         // Caller-requested host mounts (e.g. a local MCP server build the
@@ -331,7 +341,8 @@ export class DockerSandbox {
    * with an explicit mode — e.g. installing a small CLI shim onto PATH. Content
    * is staged on the host and `docker cp`'d in, so it arrives as data with no
    * shell quoting, and the copy runs as the engine, so it can write system dirs
-   * regardless of the container user. The chmod then sets the mode as root.
+   * regardless of the container user. `docker cp` preserves host ownership, so
+   * one checked root command then both chowns the file to root and sets its mode.
    */
   async writeRootFile(
     containerPath: string,
@@ -347,14 +358,37 @@ export class DockerSandbox {
     } finally {
       rmSync(staging, { recursive: true, force: true });
     }
-    const chmod = await this.runShellAsRoot(
-      `chmod ${mode} ${shellQuote(containerPath)}`
+    const secure = await this.runShellAsRoot(
+      `chown root:root ${shellQuote(containerPath)} && chmod ${mode} ${shellQuote(containerPath)}`
     );
-    if (!chmod.ok) {
-      throw new Error(`failed to chmod ${containerPath}: ${chmod.stderr}`);
+    if (!secure.ok) {
+      throw new Error(`failed to secure ${containerPath}: ${secure.stderr}`);
     }
   }
 
+  /**
+   * Read an absolute container path as root, in exec form with an absolute
+   * binary rather than a shell, so nothing on the agent's `PATH` (including
+   * `SANDBOX_PATH`'s writable first entry) can shadow the read.
+   */
+  async readRootFile(containerPath: string): Promise<string> {
+    this.assertRunning();
+    const result = await dockerCli([
+      'exec',
+      '--user',
+      'root',
+      this.containerId!,
+      '/bin/cat',
+      containerPath,
+    ]);
+    if (!result.ok) {
+      throw new Error(
+        `failed to read ${containerPath} as root: ${result.stderr}`
+      );
+    }
+    return result.stdout;
+  }
+
   private async execCommand(
     command: string,
     options: { env?: Record<string, string>; user: string; timeoutMs?: number }

packages/sandbox/src/index.ts

@@ -19,13 +19,20 @@ export {
 } from './supabase.js';
 export type { SetupSupabaseSandboxOptions } from './supabase.js';
 export {
+  buildDockerDaemonShimScript,
   buildLocalStackScoringContext,
   buildLocalStackTools,
+  buildSupabaseShimScript,
   buildToolSurfaceAddendum,
+  LOCAL_STACK_MARKER_PATH,
   localStackRuntime,
   toAgentSandbox,
 } from './local-stack-runtime.js';
-export type { LocalStackRuntimeOptions } from './local-stack-runtime.js';
+export type {
+  DockerState,
+  LocalStackEnvironmentMarker,
+  LocalStackRuntimeOptions,
+} from './local-stack-runtime.js';
 export {
   SKILLS_CLI_VERSION,
   SKILLS_INSTALL_AGENTS,