ci(cli): validate installed Eval frameworks

.github/workflows/cli-package-validation.yml

@@ -144,3 +144,36 @@ jobs:
           path: packages/cli/release
       - name: Validate the installed tarball
         run: node scripts/smoke-release-cli-package.mjs
+
+  eval:
+    name: Validate installed CLI Eval
+    needs: build
+    runs-on: ubuntu-24.04
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version: '24'
+      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
+        with:
+          python-version: '3.12'
+      - name: Select the release npm toolchain
+        run: npm install --global --no-audit --no-fund npm@11.12.1
+      - name: Install pinned Eval frameworks
+        run: |
+          python -m venv "$RUNNER_TEMP/maka-harbor"
+          "$RUNNER_TEMP/maka-harbor/bin/python" -m pip install --disable-pip-version-check 'harbor==0.20.0'
+          python -m venv "$RUNNER_TEMP/maka-pier"
+          "$RUNNER_TEMP/maka-pier/bin/python" -m pip install --disable-pip-version-check 'datacurve-pier==0.3.0'
+          echo "MAKA_RELEASE_HARBOR_PYTHON=$RUNNER_TEMP/maka-harbor/bin/python" >> "$GITHUB_ENV"
+          echo "MAKA_RELEASE_PIER_PYTHON=$RUNNER_TEMP/maka-pier/bin/python" >> "$GITHUB_ENV"
+      - name: Download the release candidate
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: cli-release-candidate
+          path: packages/cli/release
+      - name: Validate real Harbor and Pier cells
+        run: npm run release:cli:eval

package.json

@@ -44,9 +44,10 @@
     "check:cli-third-party-notices": "node scripts/generate-third-party-notices.mjs --target cli --check",
     "release:cli:pack": "node scripts/release-cli-package.mjs",
     "release:cli:smoke": "node scripts/smoke-release-cli-package.mjs",
+    "release:cli:eval": "node scripts/release-cli-eval-package.mjs",
     "generate:windows-cargo-notices": "node scripts/generate-windows-cargo-notices.mjs",
     "check:windows-cargo-notices": "node scripts/generate-windows-cargo-notices.mjs --check",
-    "check:release": "npm run check:stale && npm run check:third-party-notices && npm run check:cli-third-party-notices && node --test scripts/release-cli-file-policy.test.mjs scripts/release-cli-artifact-policy.test.mjs",
+    "check:release": "npm run check:stale && npm run check:third-party-notices && npm run check:cli-third-party-notices && node --test scripts/release-cli-file-policy.test.mjs scripts/release-cli-artifact-policy.test.mjs scripts/release-cli-eval-support.test.mjs",
     "package:macos-arm64": "node scripts/package-macos-arm64.mjs",
     "verify:macos-arm64": "node scripts/verify-macos-arm64-dmg.mjs",
     "package:windows-x64": "node scripts/package-windows-x64.mjs",

packages/eval/src/__tests__/lifecycle-boundaries.test.ts

@@ -1295,7 +1295,37 @@ test('pier cannot declare an egress proxy it never enforces', () => {
   );
 });
 
-test('launched trial environment does not inherit MAKA_EVAL_FRAMEWORK', {
+test('Pier rejects configured mounts that collide with framework log ownership', () => {
+  const root = join(tmpdir(), 'maka-test-pier-reserved-mount');
+  const restoreEnvironment = setEnvironment({
+    MAKA_TEST_MOUNT: join(root, 'mount'),
+    MAKA_TEST_PYTHON: join(root, 'python'),
+    MAKA_TEST_TASKS: join(root, 'tasks'),
+    MAKA_TEST_TRIALS: join(root, 'trials'),
+  });
+  try {
+    for (const target of ['/logs/agent/../agent', '/logs/verifier/reward.txt']) {
+      assert.throws(
+        () =>
+          createPierExecutor(
+            {
+              ...executorConfig(),
+              tasksRootEnv: 'MAKA_TEST_TASKS',
+              mounts: [{ sourceEnv: 'MAKA_TEST_MOUNT', target, readOnly: true }],
+            },
+            'experiment.json',
+          ),
+        (error) =>
+          error instanceof Error &&
+          error.message === `Pier mount target ${target} is reserved for framework logs`,
+      );
+    }
+  } finally {
+    restoreEnvironment();
+  }
+});
+
+test('Pier preserves its log mounts without inheriting MAKA_EVAL_FRAMEWORK', {
   timeout: 10_000,
 }, async () => {
   const root = await mkdtemp(join(tmpdir(), 'maka-eval-framework-env-'));
@@ -1309,6 +1339,8 @@ import { mkdir, readFile, writeFile } from 'node:fs/promises';
 const config = JSON.parse(await readFile(process.argv.at(-1), 'utf8'));
 await writeFile(process.env.MAKA_TEST_ENV, JSON.stringify({
   framework: process.env.MAKA_EVAL_FRAMEWORK ?? null,
+  mounts: config.environment.mounts,
+  trialName: config.trial_name,
 }));
 const socket = connect(config.agent.kwargs.relay_port, config.agent.kwargs.relay_host);
 socket.setEncoding('utf8');
@@ -1344,14 +1376,33 @@ socket.end();
     MAKA_TEST_PYTHON: executable,
     MAKA_TEST_TRIALS: root,
     MAKA_TEST_ENV: envDump,
+    MAKA_TEST_MOUNT: root,
+    MAKA_TEST_TASKS: root,
     MAKA_EVAL_FRAMEWORK: 'pier',
   });
   try {
+    const spec: ExperimentSpec = {
+      ...experiment(),
+      executor: {
+        kind: 'pier',
+        config: {
+          ...executorConfig(),
+          tasksRootEnv: 'MAKA_TEST_TASKS',
+          mounts: [{ sourceEnv: 'MAKA_TEST_MOUNT', target: '/input', readOnly: true }],
+        },
+      },
+      tasks: [{ id: 'task', input: 'solve', config: { pier: { path: 'task' } } }],
+    };
     const results = await runExperiment({
-      spec: experiment(),
+      spec,
       store: new FileAttemptStore(join(root, 'attempts')),
-      executor: createHarborExecutor(
-        { ...executorConfig(), preparationEnvironment: ['MAKA_TEST_ENV'] },
+      executor: createPierExecutor(
+        {
+          ...executorConfig(),
+          tasksRootEnv: 'MAKA_TEST_TASKS',
+          preparationEnvironment: ['MAKA_TEST_ENV'],
+          mounts: [{ sourceEnv: 'MAKA_TEST_MOUNT', target: '/input', readOnly: true }],
+        },
         join(root, 'experiment.json'),
       ),
       subjects: [
@@ -1372,7 +1423,26 @@ socket.end();
       ],
     });
     assert.equal(results.get('task::1::external')?.result.status, 'completed');
-    assert.deepEqual(JSON.parse(await readFile(envDump, 'utf8')), { framework: null });
+    const launched = JSON.parse(await readFile(envDump, 'utf8')) as {
+      framework: string | null;
+      mounts: Array<{ source: string; target: string }>;
+      trialName: string;
+    };
+    assert.equal(launched.framework, null);
+    assert.deepEqual(launched.mounts, [
+      { type: 'bind', source: root, target: '/input', read_only: true },
+      { type: 'bind', source: join(root, launched.trialName, 'agent'), target: '/logs/agent' },
+      {
+        type: 'bind',
+        source: join(root, launched.trialName, 'verifier'),
+        target: '/logs/verifier',
+      },
+      {
+        type: 'bind',
+        source: join(root, launched.trialName, 'artifacts'),
+        target: '/logs/artifacts',
+      },
+    ]);
   } finally {
     restoreEnvironment();
     await rm(root, { recursive: true, force: true });

packages/eval/src/harness-executor.ts

@@ -4,7 +4,7 @@ import { once } from 'node:events';
 import { createReadStream } from 'node:fs';
 import { chmod, lstat, mkdir, readFile, readdir, unlink, writeFile } from 'node:fs/promises';
 import { createServer, type Server, type Socket } from 'node:net';
-import { basename, dirname, join, relative, resolve, sep } from 'node:path';
+import { basename, dirname, join, posix, relative, resolve, sep } from 'node:path';
 import { createInterface } from 'node:readline';
 import { decodeJsonObject, type ExperimentCell, type JsonObject } from './experiment.js';
 import {
@@ -33,6 +33,12 @@ import type { EvalResult } from './result.js';
 export type HarnessFramework = 'harbor' | 'pier';
 type RelayTransportStage = 'ready' | 'execute' | 'receive' | 'decision';
 
+const PIER_FRAMEWORK_LOG_MOUNTS = Object.freeze([
+  { directory: 'agent', target: '/logs/agent' },
+  { directory: 'verifier', target: '/logs/verifier' },
+  { directory: 'artifacts', target: '/logs/artifacts' },
+]);
+
 interface RelayTransportFailure {
   readonly stage: RelayTransportStage;
   readonly category:
@@ -195,7 +201,12 @@ async function runHarnessAttempt(
           : await waitForTrial(state.child, { phase: 'completion' });
         finalizationEvidence = completed;
         if (!finalizationConfirmed(completed)) throw new Error('Trial did not finalize cleanly');
-        const verification = await readVerification(state, cell, Boolean(options.egressProxy));
+        const verification = await readVerification(
+          state,
+          cell,
+          framework,
+          Boolean(options.egressProxy),
+        );
         verificationConfirmedBeforeCancellation = !hostCancellationObserved;
         return verification;
       },
@@ -381,7 +392,7 @@ async function startTrial(
   const task = decodeTask(framework, options, cell);
   const timeoutMultiplier = positive(cell.budget.timeoutMultiplier, 'budget.timeoutMultiplier');
   const egressPaths = await resolveEgressPaths(options);
-  const environmentConfig = resolveEnvironmentConfig(options, egressPaths);
+  const environmentConfig = resolveEnvironmentConfig(options, egressPaths, framework, trialPath);
   const networkPolicyPath = egressPaths?.networkPolicyPath;
   const executionEnvironment = {
     ...UNATTENDED_EXECUTION_ENVIRONMENT,
@@ -751,6 +762,7 @@ function inspectEgressAudit(audit: Buffer): {
 async function readVerification(
   state: RelayState,
   cell: ExperimentCell,
+  framework: HarnessFramework,
   expectEgressAudit: boolean,
 ): Promise<ExecutorVerification> {
   const result = JSON.parse(await readFile(join(state.trialPath, 'result.json'), 'utf8')) as {
@@ -777,7 +789,7 @@ async function readVerification(
         failureReason: `failed to read egress audit log ${egressAuditPath}${code ? ` (${code})` : ''}`,
         artifacts: [
           { kind: 'trial', framework: cell.executor.kind, trialName: state.trialName },
-          ...(await collectedArtifactInventory(state.trialPath)),
+          ...(await collectedArtifactInventory(state.trialPath, framework)),
           { kind: 'egress-audit-unreadable', path: EGRESS_AUDIT_ARTIFACT_PATH },
         ],
       };
@@ -796,14 +808,20 @@ async function readVerification(
     failureReason: audit.failureReason ?? (score === null ? 'verifier produced no reward' : null),
     artifacts: [
       { kind: 'trial', framework: cell.executor.kind, trialName: state.trialName },
-      ...(await collectedArtifactInventory(state.trialPath)),
+      ...(await collectedArtifactInventory(state.trialPath, framework)),
       ...audit.artifacts,
     ],
   };
 }
 
-async function collectedArtifactInventory(trialPath: string): Promise<JsonObject[]> {
-  const root = join(trialPath, 'artifacts', 'logs', 'artifacts');
+async function collectedArtifactInventory(
+  trialPath: string,
+  framework: HarnessFramework,
+): Promise<JsonObject[]> {
+  const root =
+    framework === 'pier'
+      ? join(trialPath, 'artifacts')
+      : join(trialPath, 'artifacts', 'logs', 'artifacts');
   const files: JsonObject[] = [];
   const targets = [
     join(root, basename(MAKA_RUNTIME_ARTIFACT_PATH)),
@@ -910,6 +928,18 @@ function decodeHarnessOptions(value: JsonObject, framework: HarnessFramework): H
       ? { tasksRootEnv: machinePathEnv(options.tasksRootEnv, 'tasksRootEnv') }
       : {}),
   };
+  if (framework === 'pier') {
+    const reservedTargets = PIER_FRAMEWORK_LOG_MOUNTS.map((mount) => mount.target);
+    const collision = decoded.mounts.find((mount) => {
+      const target = posix.normalize(mount.target);
+      return reservedTargets.some(
+        (reserved) => target === reserved || target.startsWith(`${reserved}/`),
+      );
+    });
+    if (collision) {
+      throw new Error(`Pier mount target ${collision.target} is reserved for framework logs`);
+    }
+  }
   for (const name of [
     decoded.pythonPathEnv,
     decoded.trialsRootEnv,
@@ -980,8 +1010,22 @@ interface ResolvedEgressPaths {
 function resolveEnvironmentConfig(
   options: HarnessOptions,
   egressPaths: ResolvedEgressPaths | undefined,
+  framework: HarnessFramework,
+  trialPath: string,
 ): JsonObject {
-  const base = { ...options.environment, mounts: resolveMounts(options.mounts) };
+  const configuredMounts = resolveMounts(options.mounts);
+  const mounts =
+    framework === 'pier'
+      ? [
+          ...configuredMounts,
+          ...PIER_FRAMEWORK_LOG_MOUNTS.map(({ directory, target }) => ({
+            type: 'bind',
+            source: join(trialPath, directory),
+            target,
+          })),
+        ]
+      : configuredMounts;
+  const base = { ...options.environment, mounts };
   if (!options.egressProxy) return base;
   if (!egressPaths) throw new Error('egress proxy paths are unavailable');
   return { ...base, extra_docker_compose: [egressPaths.composePath] };