From 974e8c6d9bf2d326c2545851dd03323d20741109 Mon Sep 17 00:00:00 2001 From: M4n5ter Date: Tue, 18 Aug 2026 12:54:38 +0800 Subject: [PATCH 1/3] ci(cli): validate installed Eval frameworks Run Harbor and Pier against the same immutable CLI tarball after cross-platform spec and prerequisite checks. Use deterministic local tasks so the release gate exercises the installed relay, Docker environment, verifier result, artifacts, and cleanup without provider credentials. Preserve Pier's framework-owned log mounts when adding user mounts; otherwise an explicit empty mount list replaces the paths required for subject scope, rewards, and collected artifacts. Generated-by: OpenAI Codex --- .github/workflows/cli-package-validation.yml | 33 ++ package.json | 1 + .../__tests__/lifecycle-boundaries.test.ts | 50 ++- packages/eval/src/harness-executor.ts | 38 +- scripts/release-cli-eval-package.mjs | 400 ++++++++++++++++++ scripts/smoke-release-cli-package.mjs | 75 ++++ 6 files changed, 585 insertions(+), 12 deletions(-) create mode 100644 scripts/release-cli-eval-package.mjs diff --git a/.github/workflows/cli-package-validation.yml b/.github/workflows/cli-package-validation.yml index 40444feb72..60904b03e8 100644 --- a/.github/workflows/cli-package-validation.yml +++ b/.github/workflows/cli-package-validation.yml @@ -144,3 +144,36 @@ jobs: path: packages/cli/release - name: Validate the installed tarball run: node scripts/smoke-release-cli-package.mjs + + eval: + name: Validate installed CLI Eval + needs: build + runs-on: ubuntu-24.04 + timeout-minutes: 30 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: '24' + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.12' + - name: Select the release npm toolchain + run: npm install --global --no-audit --no-fund npm@11.12.1 + - name: Install pinned Eval frameworks + run: | + python -m venv "$RUNNER_TEMP/maka-harbor" + "$RUNNER_TEMP/maka-harbor/bin/python" -m pip install --disable-pip-version-check 'harbor==0.20.0' + python -m venv "$RUNNER_TEMP/maka-pier" + "$RUNNER_TEMP/maka-pier/bin/python" -m pip install --disable-pip-version-check 'datacurve-pier==0.3.0' + echo "MAKA_RELEASE_HARBOR_PYTHON=$RUNNER_TEMP/maka-harbor/bin/python" >> "$GITHUB_ENV" + echo "MAKA_RELEASE_PIER_PYTHON=$RUNNER_TEMP/maka-pier/bin/python" >> "$GITHUB_ENV" + - name: Download the release candidate + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: cli-release-candidate + path: packages/cli/release + - name: Validate real Harbor and Pier cells + run: npm run release:cli:eval diff --git a/package.json b/package.json index 0364a23e80..d3d0a94e76 100644 --- a/package.json +++ b/package.json @@ -44,6 +44,7 @@ "check:cli-third-party-notices": "node scripts/generate-third-party-notices.mjs --target cli --check", "release:cli:pack": "node scripts/release-cli-package.mjs", "release:cli:smoke": "node scripts/smoke-release-cli-package.mjs", + "release:cli:eval": "node scripts/release-cli-eval-package.mjs", "generate:windows-cargo-notices": "node scripts/generate-windows-cargo-notices.mjs", "check:windows-cargo-notices": "node scripts/generate-windows-cargo-notices.mjs --check", "check:release": "npm run check:stale && npm run check:third-party-notices && npm run check:cli-third-party-notices && node --test scripts/release-cli-file-policy.test.mjs scripts/release-cli-artifact-policy.test.mjs", diff --git a/packages/eval/src/__tests__/lifecycle-boundaries.test.ts b/packages/eval/src/__tests__/lifecycle-boundaries.test.ts index d612758266..b73ef11732 100644 --- a/packages/eval/src/__tests__/lifecycle-boundaries.test.ts +++ b/packages/eval/src/__tests__/lifecycle-boundaries.test.ts @@ -1295,7 +1295,7 @@ test('pier cannot declare an egress proxy it never enforces', () => { ); }); -test('launched trial environment does not inherit MAKA_EVAL_FRAMEWORK', { +test('Pier preserves its log mounts without inheriting MAKA_EVAL_FRAMEWORK', { timeout: 10_000, }, async () => { const root = await mkdtemp(join(tmpdir(), 'maka-eval-framework-env-')); @@ -1309,6 +1309,8 @@ import { mkdir, readFile, writeFile } from 'node:fs/promises'; const config = JSON.parse(await readFile(process.argv.at(-1), 'utf8')); await writeFile(process.env.MAKA_TEST_ENV, JSON.stringify({ framework: process.env.MAKA_EVAL_FRAMEWORK ?? null, + mounts: config.environment.mounts, + trialName: config.trial_name, })); const socket = connect(config.agent.kwargs.relay_port, config.agent.kwargs.relay_host); socket.setEncoding('utf8'); @@ -1344,14 +1346,33 @@ socket.end(); MAKA_TEST_PYTHON: executable, MAKA_TEST_TRIALS: root, MAKA_TEST_ENV: envDump, + MAKA_TEST_MOUNT: root, + MAKA_TEST_TASKS: root, MAKA_EVAL_FRAMEWORK: 'pier', }); try { + const spec: ExperimentSpec = { + ...experiment(), + executor: { + kind: 'pier', + config: { + ...executorConfig(), + tasksRootEnv: 'MAKA_TEST_TASKS', + mounts: [{ sourceEnv: 'MAKA_TEST_MOUNT', target: '/input', readOnly: true }], + }, + }, + tasks: [{ id: 'task', input: 'solve', config: { pier: { path: 'task' } } }], + }; const results = await runExperiment({ - spec: experiment(), + spec, store: new FileAttemptStore(join(root, 'attempts')), - executor: createHarborExecutor( - { ...executorConfig(), preparationEnvironment: ['MAKA_TEST_ENV'] }, + executor: createPierExecutor( + { + ...executorConfig(), + tasksRootEnv: 'MAKA_TEST_TASKS', + preparationEnvironment: ['MAKA_TEST_ENV'], + mounts: [{ sourceEnv: 'MAKA_TEST_MOUNT', target: '/input', readOnly: true }], + }, join(root, 'experiment.json'), ), subjects: [ @@ -1372,7 +1393,26 @@ socket.end(); ], }); assert.equal(results.get('task::1::external')?.result.status, 'completed'); - assert.deepEqual(JSON.parse(await readFile(envDump, 'utf8')), { framework: null }); + const launched = JSON.parse(await readFile(envDump, 'utf8')) as { + framework: string | null; + mounts: Array<{ source: string; target: string }>; + trialName: string; + }; + assert.equal(launched.framework, null); + assert.deepEqual(launched.mounts, [ + { type: 'bind', source: root, target: '/input', read_only: true }, + { type: 'bind', source: join(root, launched.trialName, 'agent'), target: '/logs/agent' }, + { + type: 'bind', + source: join(root, launched.trialName, 'verifier'), + target: '/logs/verifier', + }, + { + type: 'bind', + source: join(root, launched.trialName, 'artifacts'), + target: '/logs/artifacts', + }, + ]); } finally { restoreEnvironment(); await rm(root, { recursive: true, force: true }); diff --git a/packages/eval/src/harness-executor.ts b/packages/eval/src/harness-executor.ts index dbdee59f66..9fa596d79b 100644 --- a/packages/eval/src/harness-executor.ts +++ b/packages/eval/src/harness-executor.ts @@ -195,7 +195,12 @@ async function runHarnessAttempt( : await waitForTrial(state.child, { phase: 'completion' }); finalizationEvidence = completed; if (!finalizationConfirmed(completed)) throw new Error('Trial did not finalize cleanly'); - const verification = await readVerification(state, cell, Boolean(options.egressProxy)); + const verification = await readVerification( + state, + cell, + framework, + Boolean(options.egressProxy), + ); verificationConfirmedBeforeCancellation = !hostCancellationObserved; return verification; }, @@ -381,7 +386,7 @@ async function startTrial( const task = decodeTask(framework, options, cell); const timeoutMultiplier = positive(cell.budget.timeoutMultiplier, 'budget.timeoutMultiplier'); const egressPaths = await resolveEgressPaths(options); - const environmentConfig = resolveEnvironmentConfig(options, egressPaths); + const environmentConfig = resolveEnvironmentConfig(options, egressPaths, framework, trialPath); const networkPolicyPath = egressPaths?.networkPolicyPath; const executionEnvironment = { ...UNATTENDED_EXECUTION_ENVIRONMENT, @@ -751,6 +756,7 @@ function inspectEgressAudit(audit: Buffer): { async function readVerification( state: RelayState, cell: ExperimentCell, + framework: HarnessFramework, expectEgressAudit: boolean, ): Promise { const result = JSON.parse(await readFile(join(state.trialPath, 'result.json'), 'utf8')) as { @@ -777,7 +783,7 @@ async function readVerification( failureReason: `failed to read egress audit log ${egressAuditPath}${code ? ` (${code})` : ''}`, artifacts: [ { kind: 'trial', framework: cell.executor.kind, trialName: state.trialName }, - ...(await collectedArtifactInventory(state.trialPath)), + ...(await collectedArtifactInventory(state.trialPath, framework)), { kind: 'egress-audit-unreadable', path: EGRESS_AUDIT_ARTIFACT_PATH }, ], }; @@ -796,14 +802,20 @@ async function readVerification( failureReason: audit.failureReason ?? (score === null ? 'verifier produced no reward' : null), artifacts: [ { kind: 'trial', framework: cell.executor.kind, trialName: state.trialName }, - ...(await collectedArtifactInventory(state.trialPath)), + ...(await collectedArtifactInventory(state.trialPath, framework)), ...audit.artifacts, ], }; } -async function collectedArtifactInventory(trialPath: string): Promise { - const root = join(trialPath, 'artifacts', 'logs', 'artifacts'); +async function collectedArtifactInventory( + trialPath: string, + framework: HarnessFramework, +): Promise { + const root = + framework === 'pier' + ? join(trialPath, 'artifacts') + : join(trialPath, 'artifacts', 'logs', 'artifacts'); const files: JsonObject[] = []; const targets = [ join(root, basename(MAKA_RUNTIME_ARTIFACT_PATH)), @@ -980,8 +992,20 @@ interface ResolvedEgressPaths { function resolveEnvironmentConfig( options: HarnessOptions, egressPaths: ResolvedEgressPaths | undefined, + framework: HarnessFramework, + trialPath: string, ): JsonObject { - const base = { ...options.environment, mounts: resolveMounts(options.mounts) }; + const configuredMounts = resolveMounts(options.mounts); + const mounts = + framework === 'pier' + ? [ + ...configuredMounts, + { type: 'bind', source: join(trialPath, 'agent'), target: '/logs/agent' }, + { type: 'bind', source: join(trialPath, 'verifier'), target: '/logs/verifier' }, + { type: 'bind', source: join(trialPath, 'artifacts'), target: '/logs/artifacts' }, + ] + : configuredMounts; + const base = { ...options.environment, mounts }; if (!options.egressProxy) return base; if (!egressPaths) throw new Error('egress proxy paths are unavailable'); return { ...base, extra_docker_compose: [egressPaths.composePath] }; diff --git a/scripts/release-cli-eval-package.mjs b/scripts/release-cli-eval-package.mjs new file mode 100644 index 0000000000..3e03fdea3e --- /dev/null +++ b/scripts/release-cli-eval-package.mjs @@ -0,0 +1,400 @@ +import { createHash } from 'node:crypto'; +import { + chmodSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { basename, join, relative, resolve, sep } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { execFileSync } from 'node:child_process'; + +const HARBOR_VERSION = '0.20.0'; +const PIER_VERSION = '0.3.0'; +const TASK_IMAGE = + 'python:3.12-slim@sha256:dd29372629eeba2dd003fd9e9d35a5b8236c44727875a0364254b5127af88e65'; +const PROCESS_TIMEOUT_MS = 10 * 60 * 1000; +const MAX_OUTPUT_BYTES = 2 * 1024 * 1024; +const releaseDirectory = resolve('packages/cli/release'); +const tarballPath = findReleaseTarball(); + +if (process.platform !== 'linux' || process.arch !== 'x64') { + throw new Error('The real Eval release validation requires Linux x64'); +} + +const root = mkdtempSync(join(tmpdir(), 'maka-cli-eval-validation-')); +let primaryError; +try { + validateChecksum(); + const prefix = join(root, 'prefix'); + const environment = isolatedEnvironment(join(root, 'home')); + logStep('installing the immutable tarball with an empty offline npm cache'); + run( + 'npm', + [ + 'install', + '--global', + '--ignore-scripts', + '--offline', + '--no-audit', + '--no-fund', + '--cache', + join(root, 'npm-cache'), + '--prefix', + prefix, + tarballPath, + ], + environment, + root, + ); + + const maka = join(prefix, 'bin/maka'); + const packageRoot = join(prefix, 'lib/node_modules/maka-agent'); + if (!existsSync(maka) || !existsSync(join(packageRoot, 'packages/eval/harbor/run_trial.py'))) { + throw new Error('The installed candidate is missing its CLI or bundled Eval runtime'); + } + + const fixture = createTaskFixture(join(root, 'fixture')); + for (const framework of ['harbor', 'pier']) { + logStep(`running a real ${framework} Docker cell from the installed candidate`); + validateFramework({ + environment, + fixture, + framework, + maka, + root: join(root, framework), + }); + } + logStep(`OK — installed ${basename(tarballPath)} completed real Harbor and Pier cells`); +} catch (error) { + primaryError = error; +} finally { + let cleanupError; + try { + rmSync(root, { recursive: true, force: true, maxRetries: 5, retryDelay: 200 }); + } catch (error) { + cleanupError = error; + } + if (primaryError && cleanupError) { + throw new AggregateError([primaryError, cleanupError], 'Eval validation and cleanup failed'); + } + if (primaryError) throw primaryError; + if (cleanupError) throw cleanupError; +} + +function validateFramework({ environment, fixture, framework, maka, root: frameworkRoot }) { + mkdirSync(frameworkRoot, { recursive: true, mode: 0o700 }); + const outputRoot = join(frameworkRoot, 'output'); + const trialsRoot = join(frameworkRoot, 'trials'); + const pythonVariable = + framework === 'harbor' ? 'MAKA_RELEASE_HARBOR_PYTHON' : 'MAKA_RELEASE_PIER_PYTHON'; + const pythonPath = process.env[pythonVariable]; + if (!pythonPath) throw new Error(`${pythonVariable} is required`); + const pythonPathVariable = `MAKA_RELEASE_${framework.toUpperCase()}_PYTHON_PATH`; + const trialsVariable = `MAKA_RELEASE_${framework.toUpperCase()}_TRIALS`; + const tasksVariable = 'MAKA_RELEASE_PIER_TASKS'; + const specPath = join(frameworkRoot, 'experiment.json'); + const spec = experimentSpec({ + fixture, + framework, + pythonPathVariable, + tasksVariable, + trialsVariable, + }); + writeFileSync(specPath, `${JSON.stringify(spec)}\n`, { mode: 0o600 }); + const childEnvironment = { + ...environment, + [pythonPathVariable]: pythonPath, + [trialsVariable]: trialsRoot, + ...(framework === 'pier' ? { [tasksVariable]: fixture.tasksRoot } : {}), + }; + const invocation = runAllowingFailure( + maka, + ['eval', 'run', specPath, '--out', outputRoot], + childEnvironment, + frameworkRoot, + ); + const summary = JSON.parse(invocation.stdout.trim()); + const attempts = findAttemptFiles(join(outputRoot, 'attempts')); + if (attempts.length !== 1) { + throw new Error(`${framework} produced ${attempts.length} attempt files instead of one`); + } + const attempt = JSON.parse(readFileSync(attempts[0], 'utf8')); + const result = attempt.result; + if ( + invocation.status !== 0 || + summary.experimentId !== `release-${framework}` || + summary.cells !== 1 || + summary.incomplete !== 0 || + attempt.cellId !== 'task::1::subject' || + attempt.sequence !== 1 || + result?.status !== 'completed' || + result?.score !== 1 || + result?.usage !== null || + result?.costUsd !== null || + result?.failureReason !== null + ) { + throw new Error( + `${framework} produced an invalid completed attempt: ${JSON.stringify({ + invocation, + summary, + attempt, + frameworkDiagnostics: findJsonDiagnostics(trialsRoot), + })}`, + ); + } + const trialArtifact = result.artifacts.find( + (artifact) => artifact.kind === 'trial' && artifact.framework === framework, + ); + const processArtifact = result.artifacts.find( + (artifact) => artifact.kind === 'external_process' && artifact.exitCode === 0, + ); + const collected = result.artifacts.filter((artifact) => artifact.kind === 'collected-artifact'); + if (!trialArtifact || !processArtifact || collected.length < 2) { + throw new Error(`${framework} did not preserve the expected trial and subject artifacts`); + } + const containers = run( + 'docker', + ['ps', '--all', '--quiet', '--filter', `name=${trialArtifact.trialName}`], + childEnvironment, + frameworkRoot, + ).trim(); + if (containers) throw new Error(`${framework} left trial containers behind: ${containers}`); +} + +function experimentSpec({ fixture, framework, pythonPathVariable, tasksVariable, trialsVariable }) { + return { + schemaVersion: 'maka.eval.v1', + id: `release-${framework}`, + benchmark: { + id: 'release-validation', + version: fixture.commit, + config: { repository: fixture.repository }, + }, + executor: { + kind: framework, + config: { + frameworkVersion: framework === 'harbor' ? HARBOR_VERSION : PIER_VERSION, + pythonPathEnv: pythonPathVariable, + trialsRootEnv: trialsVariable, + ...(framework === 'pier' ? { tasksRootEnv: tasksVariable } : {}), + environment: { type: 'docker', delete: true }, + preparationEnvironment: [], + mounts: [], + }, + }, + subjects: [ + { + id: 'subject', + kind: 'external', + credentials: [], + config: { command: '/bin/true', args: [], result: 'exit-code' }, + }, + ], + tasks: [ + { + id: 'task', + input: 'Exit successfully without modifying the task.', + config: framework === 'harbor' ? { harbor: { path: 'task' } } : { pier: { path: 'task' } }, + }, + ], + repetitions: 1, + budget: { timeoutMultiplier: 1 }, + verifier: { reward: 'reward' }, + }; +} + +function createTaskFixture(root) { + const repositoryRoot = join(root, 'repository'); + const taskRoot = join(repositoryRoot, 'task'); + mkdirSync(join(taskRoot, 'environment'), { recursive: true, mode: 0o700 }); + mkdirSync(join(taskRoot, 'tests'), { recursive: true, mode: 0o700 }); + writeFileSync( + join(taskRoot, 'task.toml'), + [ + 'version = "1.0"', + '', + '[metadata]', + '', + '[verifier]', + 'timeout_sec = 60.0', + '', + '[agent]', + 'timeout_sec = 60.0', + '', + '[environment]', + 'build_timeout_sec = 120.0', + '', + ].join('\n'), + { mode: 0o600 }, + ); + writeFileSync( + join(taskRoot, 'instruction.md'), + 'Exit successfully without modifying the task.\n', + { mode: 0o600 }, + ); + writeFileSync(join(taskRoot, 'environment/Dockerfile'), `FROM ${TASK_IMAGE}\nWORKDIR /app\n`, { + mode: 0o600, + }); + const testPath = join(taskRoot, 'tests/test.sh'); + writeFileSync(testPath, '#!/bin/sh\nset -eu\nprintf "1\\n" > /logs/verifier/reward.txt\n', { + mode: 0o700, + }); + chmodSync(testPath, 0o700); + run('git', ['init', '--initial-branch=main'], process.env, repositoryRoot); + run('git', ['config', 'user.name', 'Maka Release Validation'], process.env, repositoryRoot); + run( + 'git', + ['config', 'user.email', 'release-validation@maka.invalid'], + process.env, + repositoryRoot, + ); + run('git', ['add', '.'], process.env, repositoryRoot); + run( + 'git', + ['commit', '--message', 'Add deterministic release validation task'], + process.env, + repositoryRoot, + ); + const commit = run('git', ['rev-parse', 'HEAD'], process.env, repositoryRoot).trim(); + return { + commit, + repository: pathToFileURL(repositoryRoot).href, + tasksRoot: repositoryRoot, + }; +} + +function findAttemptFiles(root) { + if (!existsSync(root)) return []; + const files = []; + for (const entry of readdirSync(root, { withFileTypes: true })) { + const path = join(root, entry.name); + if (entry.isDirectory()) files.push(...findAttemptFiles(path)); + else if (/^\d{6}\.json$/u.test(entry.name)) files.push(path); + } + return files.sort(); +} + +function findJsonDiagnostics(root, current = root) { + if (!existsSync(current)) return {}; + const diagnostics = {}; + for (const entry of readdirSync(current, { withFileTypes: true })) { + const path = join(current, entry.name); + if (entry.isDirectory()) { + Object.assign(diagnostics, findJsonDiagnostics(root, path)); + continue; + } + if (!['preparation-error.json', 'result.json', 'trial.log'].includes(entry.name)) continue; + const name = relative(root, path).split(sep).join('/'); + const content = readFileSync(path, 'utf8').trim(); + if (entry.name === 'trial.log') { + diagnostics[name] = content.slice(-4_000); + continue; + } + const parsed = JSON.parse(content); + diagnostics[name] = + entry.name === 'result.json' + ? { + exceptionInfo: parsed.exception_info + ? { + type: parsed.exception_info.exception_type, + message: String(parsed.exception_info.exception_message ?? '').slice(-4_000), + } + : null, + verifierResult: parsed.verifier_result ?? null, + } + : parsed; + } + return diagnostics; +} + +function findReleaseTarball() { + const tarballs = readdirSync(releaseDirectory) + .filter((name) => /^maka-agent-[^/]+\.tgz$/u.test(name)) + .map((name) => join(releaseDirectory, name)); + if (tarballs.length !== 1) { + throw new Error( + `Expected one release tarball in ${releaseDirectory}, found ${tarballs.length}`, + ); + } + return tarballs[0]; +} + +function validateChecksum() { + const checksumPath = `${tarballPath}.sha256`; + const [expected, name, extra] = readFileSync(checksumPath, 'utf8').trim().split(/\s+/u); + if (!expected || name !== basename(tarballPath) || extra !== undefined) { + throw new Error(`Invalid checksum file ${checksumPath}`); + } + const actual = createHash('sha256').update(readFileSync(tarballPath)).digest('hex'); + if (actual !== expected) throw new Error(`Checksum mismatch for ${tarballPath}`); +} + +function isolatedEnvironment(home) { + mkdirSync(home, { recursive: true, mode: 0o700 }); + const environment = { + ...process.env, + HOME: home, + NODE_PATH: '', + XDG_CACHE_HOME: join(home, '.cache'), + XDG_CONFIG_HOME: join(home, '.config'), + XDG_DATA_HOME: join(home, '.local/share'), + }; + delete environment.PYTHONHOME; + delete environment.PYTHONPATH; + for (const name of Object.keys(environment)) { + if (name.startsWith('MAKA_EVAL_')) delete environment[name]; + } + for (const name of [ + 'ANTHROPIC_API_KEY', + 'DEEPSEEK_API_KEY', + 'OPENAI_API_KEY', + 'OPENROUTER_API_KEY', + ]) { + delete environment[name]; + } + return environment; +} + +function run(command, args, environment, cwd) { + try { + return execFileSync(command, args, { + cwd, + env: environment, + encoding: 'utf8', + timeout: PROCESS_TIMEOUT_MS, + maxBuffer: MAX_OUTPUT_BYTES, + stdio: ['ignore', 'pipe', 'pipe'], + }); + } catch (error) { + const stdout = String(error.stdout ?? '').trim(); + const stderr = String(error.stderr ?? '').trim(); + throw new Error( + `${command} ${args.join(' ')} failed${stdout ? `\nstdout:\n${stdout}` : ''}${stderr ? `\nstderr:\n${stderr}` : ''}`, + { cause: error }, + ); + } +} + +function runAllowingFailure(command, args, environment, cwd) { + try { + return { status: 0, stdout: run(command, args, environment, cwd), stderr: '' }; + } catch (error) { + const cause = error.cause; + if (typeof cause?.status !== 'number') throw error; + return { + status: cause.status, + stdout: String(cause.stdout ?? ''), + stderr: String(cause.stderr ?? ''), + }; + } +} + +function logStep(message) { + console.log(`[release-cli-eval-validation] ${message}`); +} diff --git a/scripts/smoke-release-cli-package.mjs b/scripts/smoke-release-cli-package.mjs index 0037b721da..5d3d03112b 100644 --- a/scripts/smoke-release-cli-package.mjs +++ b/scripts/smoke-release-cli-package.mjs @@ -143,6 +143,9 @@ async function validateInstalledProduct(root) { ); validateInstalledRuntimeFiles(packageRoot); + logStep('checking installed Eval spec decoding and framework preflight'); + smokeEvalPreflight({ crossSpawn: crossSpawn.sync, environment: baseEnvironment, maka, root }); + logStep('checking installed native PTY and file-lock modules'); const nodePty = await importInstalled(packageRoot, 'node_modules/node-pty/lib/index.js'); const ptySpawn = nodePty.spawn ?? nodePty.default?.spawn; @@ -230,6 +233,78 @@ function validateInstalledRuntimeFiles(packageRoot) { ); } +function smokeEvalPreflight({ crossSpawn, environment, maka, root }) { + const evalRoot = join(root, 'eval-preflight'); + mkdirSync(evalRoot, { recursive: true }); + const machineEnvironment = { + ...environment, + MAKA_RELEASE_EVAL_PYTHON: process.execPath, + MAKA_RELEASE_EVAL_TASKS: evalRoot, + MAKA_RELEASE_EVAL_TRIALS: join(evalRoot, 'trials'), + }; + for (const framework of ['harbor', 'pier']) { + const specPath = join(evalRoot, `${framework}.json`); + writeFileSync(specPath, `${JSON.stringify(evalPreflightSpec(framework))}\n`, 'utf8'); + const result = crossSpawn(maka, ['eval', 'run', specPath, '--out', join(evalRoot, framework)], { + cwd: evalRoot, + env: machineEnvironment, + encoding: 'utf8', + timeout: 30_000, + maxBuffer: MAX_OUTPUT_BYTES, + }); + if (result.error) throw result.error; + if (result.status === 0) { + throw new Error(`${framework} Eval preflight unexpectedly accepted the Node executable`); + } + assertOutput( + `${result.stdout ?? ''}\n${result.stderr ?? ''}`, + `${framework} Python environment MAKA_RELEASE_EVAL_PYTHON is unavailable or does not provide`, + ); + } +} + +function evalPreflightSpec(framework) { + return { + schemaVersion: 'maka.eval.v1', + id: `release-${framework}-preflight`, + benchmark: { + id: 'release-preflight', + version: '0000000000000000000000000000000000000000', + config: { repository: 'https://invalid.invalid/release-preflight.git' }, + }, + executor: { + kind: framework, + config: { + frameworkVersion: framework === 'harbor' ? '0.20.0' : '0.3.0', + pythonPathEnv: 'MAKA_RELEASE_EVAL_PYTHON', + trialsRootEnv: 'MAKA_RELEASE_EVAL_TRIALS', + ...(framework === 'pier' ? { tasksRootEnv: 'MAKA_RELEASE_EVAL_TASKS' } : {}), + environment: { type: 'docker', delete: true }, + preparationEnvironment: [], + mounts: [], + }, + }, + subjects: [ + { + id: 'subject', + kind: 'external', + credentials: [], + config: { command: process.execPath, args: ['--version'], result: 'exit-code' }, + }, + ], + tasks: [ + { + id: 'task', + input: 'Do not execute this preflight-only task.', + config: framework === 'harbor' ? { harbor: { path: 'task' } } : { pier: { path: 'task' } }, + }, + ], + repetitions: 1, + budget: { timeoutMultiplier: 1 }, + verifier: { reward: 'reward' }, + }; +} + async function smokePty(ptySpawn, environment, cwd) { const result = await runPtyScenario({ ptySpawn, From 1845005cddda9a8b01198f0b6e6fe3b2ea751d3c Mon Sep 17 00:00:00 2001 From: M4n5ter Date: Tue, 18 Aug 2026 13:47:46 +0800 Subject: [PATCH 2/3] fix(eval): harden release validation failures Reserve Pier's framework-owned log subtrees before Docker composition so configured mounts cannot shadow verifier rewards or collected artifacts. Keep framework failures authoritative when summary or diagnostic evidence is malformed or unreadable, and create the deterministic Git fixture under the same isolated environment as the installed candidate. Generated-by: OpenAI Codex --- package.json | 2 +- .../__tests__/lifecycle-boundaries.test.ts | 28 ++ packages/eval/src/harness-executor.ts | 28 +- scripts/release-cli-eval-package.mjs | 223 +----------- scripts/release-cli-eval-support.mjs | 325 ++++++++++++++++++ scripts/release-cli-eval-support.test.mjs | 131 +++++++ 6 files changed, 528 insertions(+), 209 deletions(-) create mode 100644 scripts/release-cli-eval-support.mjs create mode 100644 scripts/release-cli-eval-support.test.mjs diff --git a/package.json b/package.json index d3d0a94e76..e9dd8f6d26 100644 --- a/package.json +++ b/package.json @@ -47,7 +47,7 @@ "release:cli:eval": "node scripts/release-cli-eval-package.mjs", "generate:windows-cargo-notices": "node scripts/generate-windows-cargo-notices.mjs", "check:windows-cargo-notices": "node scripts/generate-windows-cargo-notices.mjs --check", - "check:release": "npm run check:stale && npm run check:third-party-notices && npm run check:cli-third-party-notices && node --test scripts/release-cli-file-policy.test.mjs scripts/release-cli-artifact-policy.test.mjs", + "check:release": "npm run check:stale && npm run check:third-party-notices && npm run check:cli-third-party-notices && node --test scripts/release-cli-file-policy.test.mjs scripts/release-cli-artifact-policy.test.mjs scripts/release-cli-eval-support.test.mjs", "package:macos-arm64": "node scripts/package-macos-arm64.mjs", "verify:macos-arm64": "node scripts/verify-macos-arm64-dmg.mjs", "package:windows-x64": "node scripts/package-windows-x64.mjs", diff --git a/packages/eval/src/__tests__/lifecycle-boundaries.test.ts b/packages/eval/src/__tests__/lifecycle-boundaries.test.ts index b73ef11732..d049fa1015 100644 --- a/packages/eval/src/__tests__/lifecycle-boundaries.test.ts +++ b/packages/eval/src/__tests__/lifecycle-boundaries.test.ts @@ -1295,6 +1295,34 @@ test('pier cannot declare an egress proxy it never enforces', () => { ); }); +test('Pier rejects configured mounts that collide with framework log ownership', () => { + const root = join(tmpdir(), 'maka-test-pier-reserved-mount'); + const restoreEnvironment = setEnvironment({ + MAKA_TEST_MOUNT: join(root, 'mount'), + MAKA_TEST_PYTHON: join(root, 'python'), + MAKA_TEST_TASKS: join(root, 'tasks'), + MAKA_TEST_TRIALS: join(root, 'trials'), + }); + try { + for (const target of ['/logs/agent/../agent', '/logs/verifier/reward.txt']) { + assert.throws( + () => + createPierExecutor( + { + ...executorConfig(), + tasksRootEnv: 'MAKA_TEST_TASKS', + mounts: [{ sourceEnv: 'MAKA_TEST_MOUNT', target, readOnly: true }], + }, + 'experiment.json', + ), + new RegExp(`Pier mount target ${target.replaceAll('/', '\\/')} is reserved`, 'u'), + ); + } + } finally { + restoreEnvironment(); + } +}); + test('Pier preserves its log mounts without inheriting MAKA_EVAL_FRAMEWORK', { timeout: 10_000, }, async () => { diff --git a/packages/eval/src/harness-executor.ts b/packages/eval/src/harness-executor.ts index 9fa596d79b..99e71c5344 100644 --- a/packages/eval/src/harness-executor.ts +++ b/packages/eval/src/harness-executor.ts @@ -4,7 +4,7 @@ import { once } from 'node:events'; import { createReadStream } from 'node:fs'; import { chmod, lstat, mkdir, readFile, readdir, unlink, writeFile } from 'node:fs/promises'; import { createServer, type Server, type Socket } from 'node:net'; -import { basename, dirname, join, relative, resolve, sep } from 'node:path'; +import { basename, dirname, join, posix, relative, resolve, sep } from 'node:path'; import { createInterface } from 'node:readline'; import { decodeJsonObject, type ExperimentCell, type JsonObject } from './experiment.js'; import { @@ -33,6 +33,12 @@ import type { EvalResult } from './result.js'; export type HarnessFramework = 'harbor' | 'pier'; type RelayTransportStage = 'ready' | 'execute' | 'receive' | 'decision'; +const PIER_FRAMEWORK_LOG_MOUNTS = Object.freeze([ + { directory: 'agent', target: '/logs/agent' }, + { directory: 'verifier', target: '/logs/verifier' }, + { directory: 'artifacts', target: '/logs/artifacts' }, +]); + interface RelayTransportFailure { readonly stage: RelayTransportStage; readonly category: @@ -922,6 +928,18 @@ function decodeHarnessOptions(value: JsonObject, framework: HarnessFramework): H ? { tasksRootEnv: machinePathEnv(options.tasksRootEnv, 'tasksRootEnv') } : {}), }; + if (framework === 'pier') { + const reservedTargets = PIER_FRAMEWORK_LOG_MOUNTS.map((mount) => mount.target); + const collision = decoded.mounts.find((mount) => { + const target = posix.normalize(mount.target); + return reservedTargets.some( + (reserved) => target === reserved || target.startsWith(`${reserved}/`), + ); + }); + if (collision) { + throw new Error(`Pier mount target ${collision.target} is reserved for framework logs`); + } + } for (const name of [ decoded.pythonPathEnv, decoded.trialsRootEnv, @@ -1000,9 +1018,11 @@ function resolveEnvironmentConfig( framework === 'pier' ? [ ...configuredMounts, - { type: 'bind', source: join(trialPath, 'agent'), target: '/logs/agent' }, - { type: 'bind', source: join(trialPath, 'verifier'), target: '/logs/verifier' }, - { type: 'bind', source: join(trialPath, 'artifacts'), target: '/logs/artifacts' }, + ...PIER_FRAMEWORK_LOG_MOUNTS.map(({ directory, target }) => ({ + type: 'bind', + source: join(trialPath, directory), + target, + })), ] : configuredMounts; const base = { ...options.environment, mounts }; diff --git a/scripts/release-cli-eval-package.mjs b/scripts/release-cli-eval-package.mjs index 3e03fdea3e..82d3f6b67c 100644 --- a/scripts/release-cli-eval-package.mjs +++ b/scripts/release-cli-eval-package.mjs @@ -1,31 +1,23 @@ import { createHash } from 'node:crypto'; -import { - chmodSync, - existsSync, - mkdirSync, - mkdtempSync, - readFileSync, - readdirSync, - rmSync, - writeFileSync, -} from 'node:fs'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; -import { basename, join, relative, resolve, sep } from 'node:path'; -import { pathToFileURL } from 'node:url'; -import { execFileSync } from 'node:child_process'; +import { basename, join, resolve } from 'node:path'; +import { + createTaskFixture, + findEvalReleaseTarball, + findJsonDiagnostics, + isolatedEnvironment, + readFrameworkOutputs, + run, + runAllowingFailure, +} from './release-cli-eval-support.mjs'; const HARBOR_VERSION = '0.20.0'; const PIER_VERSION = '0.3.0'; const TASK_IMAGE = 'python:3.12-slim@sha256:dd29372629eeba2dd003fd9e9d35a5b8236c44727875a0364254b5127af88e65'; -const PROCESS_TIMEOUT_MS = 10 * 60 * 1000; -const MAX_OUTPUT_BYTES = 2 * 1024 * 1024; const releaseDirectory = resolve('packages/cli/release'); -const tarballPath = findReleaseTarball(); - -if (process.platform !== 'linux' || process.arch !== 'x64') { - throw new Error('The real Eval release validation requires Linux x64'); -} +const tarballPath = findEvalReleaseTarball(releaseDirectory); const root = mkdtempSync(join(tmpdir(), 'maka-cli-eval-validation-')); let primaryError; @@ -59,7 +51,7 @@ try { throw new Error('The installed candidate is missing its CLI or bundled Eval runtime'); } - const fixture = createTaskFixture(join(root, 'fixture')); + const fixture = createTaskFixture(join(root, 'fixture'), environment, TASK_IMAGE); for (const framework of ['harbor', 'pier']) { logStep(`running a real ${framework} Docker cell from the installed candidate`); validateFramework({ @@ -119,15 +111,14 @@ function validateFramework({ environment, fixture, framework, maka, root: framew childEnvironment, frameworkRoot, ); - const summary = JSON.parse(invocation.stdout.trim()); - const attempts = findAttemptFiles(join(outputRoot, 'attempts')); - if (attempts.length !== 1) { - throw new Error(`${framework} produced ${attempts.length} attempt files instead of one`); - } - const attempt = JSON.parse(readFileSync(attempts[0], 'utf8')); + const { summary, attempt } = readFrameworkOutputs({ + framework, + invocation, + outputRoot, + trialsRoot, + }); const result = attempt.result; if ( - invocation.status !== 0 || summary.experimentId !== `release-${framework}` || summary.cells !== 1 || summary.incomplete !== 0 || @@ -209,122 +200,6 @@ function experimentSpec({ fixture, framework, pythonPathVariable, tasksVariable, }; } -function createTaskFixture(root) { - const repositoryRoot = join(root, 'repository'); - const taskRoot = join(repositoryRoot, 'task'); - mkdirSync(join(taskRoot, 'environment'), { recursive: true, mode: 0o700 }); - mkdirSync(join(taskRoot, 'tests'), { recursive: true, mode: 0o700 }); - writeFileSync( - join(taskRoot, 'task.toml'), - [ - 'version = "1.0"', - '', - '[metadata]', - '', - '[verifier]', - 'timeout_sec = 60.0', - '', - '[agent]', - 'timeout_sec = 60.0', - '', - '[environment]', - 'build_timeout_sec = 120.0', - '', - ].join('\n'), - { mode: 0o600 }, - ); - writeFileSync( - join(taskRoot, 'instruction.md'), - 'Exit successfully without modifying the task.\n', - { mode: 0o600 }, - ); - writeFileSync(join(taskRoot, 'environment/Dockerfile'), `FROM ${TASK_IMAGE}\nWORKDIR /app\n`, { - mode: 0o600, - }); - const testPath = join(taskRoot, 'tests/test.sh'); - writeFileSync(testPath, '#!/bin/sh\nset -eu\nprintf "1\\n" > /logs/verifier/reward.txt\n', { - mode: 0o700, - }); - chmodSync(testPath, 0o700); - run('git', ['init', '--initial-branch=main'], process.env, repositoryRoot); - run('git', ['config', 'user.name', 'Maka Release Validation'], process.env, repositoryRoot); - run( - 'git', - ['config', 'user.email', 'release-validation@maka.invalid'], - process.env, - repositoryRoot, - ); - run('git', ['add', '.'], process.env, repositoryRoot); - run( - 'git', - ['commit', '--message', 'Add deterministic release validation task'], - process.env, - repositoryRoot, - ); - const commit = run('git', ['rev-parse', 'HEAD'], process.env, repositoryRoot).trim(); - return { - commit, - repository: pathToFileURL(repositoryRoot).href, - tasksRoot: repositoryRoot, - }; -} - -function findAttemptFiles(root) { - if (!existsSync(root)) return []; - const files = []; - for (const entry of readdirSync(root, { withFileTypes: true })) { - const path = join(root, entry.name); - if (entry.isDirectory()) files.push(...findAttemptFiles(path)); - else if (/^\d{6}\.json$/u.test(entry.name)) files.push(path); - } - return files.sort(); -} - -function findJsonDiagnostics(root, current = root) { - if (!existsSync(current)) return {}; - const diagnostics = {}; - for (const entry of readdirSync(current, { withFileTypes: true })) { - const path = join(current, entry.name); - if (entry.isDirectory()) { - Object.assign(diagnostics, findJsonDiagnostics(root, path)); - continue; - } - if (!['preparation-error.json', 'result.json', 'trial.log'].includes(entry.name)) continue; - const name = relative(root, path).split(sep).join('/'); - const content = readFileSync(path, 'utf8').trim(); - if (entry.name === 'trial.log') { - diagnostics[name] = content.slice(-4_000); - continue; - } - const parsed = JSON.parse(content); - diagnostics[name] = - entry.name === 'result.json' - ? { - exceptionInfo: parsed.exception_info - ? { - type: parsed.exception_info.exception_type, - message: String(parsed.exception_info.exception_message ?? '').slice(-4_000), - } - : null, - verifierResult: parsed.verifier_result ?? null, - } - : parsed; - } - return diagnostics; -} - -function findReleaseTarball() { - const tarballs = readdirSync(releaseDirectory) - .filter((name) => /^maka-agent-[^/]+\.tgz$/u.test(name)) - .map((name) => join(releaseDirectory, name)); - if (tarballs.length !== 1) { - throw new Error( - `Expected one release tarball in ${releaseDirectory}, found ${tarballs.length}`, - ); - } - return tarballs[0]; -} - function validateChecksum() { const checksumPath = `${tarballPath}.sha256`; const [expected, name, extra] = readFileSync(checksumPath, 'utf8').trim().split(/\s+/u); @@ -335,66 +210,6 @@ function validateChecksum() { if (actual !== expected) throw new Error(`Checksum mismatch for ${tarballPath}`); } -function isolatedEnvironment(home) { - mkdirSync(home, { recursive: true, mode: 0o700 }); - const environment = { - ...process.env, - HOME: home, - NODE_PATH: '', - XDG_CACHE_HOME: join(home, '.cache'), - XDG_CONFIG_HOME: join(home, '.config'), - XDG_DATA_HOME: join(home, '.local/share'), - }; - delete environment.PYTHONHOME; - delete environment.PYTHONPATH; - for (const name of Object.keys(environment)) { - if (name.startsWith('MAKA_EVAL_')) delete environment[name]; - } - for (const name of [ - 'ANTHROPIC_API_KEY', - 'DEEPSEEK_API_KEY', - 'OPENAI_API_KEY', - 'OPENROUTER_API_KEY', - ]) { - delete environment[name]; - } - return environment; -} - -function run(command, args, environment, cwd) { - try { - return execFileSync(command, args, { - cwd, - env: environment, - encoding: 'utf8', - timeout: PROCESS_TIMEOUT_MS, - maxBuffer: MAX_OUTPUT_BYTES, - stdio: ['ignore', 'pipe', 'pipe'], - }); - } catch (error) { - const stdout = String(error.stdout ?? '').trim(); - const stderr = String(error.stderr ?? '').trim(); - throw new Error( - `${command} ${args.join(' ')} failed${stdout ? `\nstdout:\n${stdout}` : ''}${stderr ? `\nstderr:\n${stderr}` : ''}`, - { cause: error }, - ); - } -} - -function runAllowingFailure(command, args, environment, cwd) { - try { - return { status: 0, stdout: run(command, args, environment, cwd), stderr: '' }; - } catch (error) { - const cause = error.cause; - if (typeof cause?.status !== 'number') throw error; - return { - status: cause.status, - stdout: String(cause.stdout ?? ''), - stderr: String(cause.stderr ?? ''), - }; - } -} - function logStep(message) { console.log(`[release-cli-eval-validation] ${message}`); } diff --git a/scripts/release-cli-eval-support.mjs b/scripts/release-cli-eval-support.mjs new file mode 100644 index 0000000000..57e50a4f4d --- /dev/null +++ b/scripts/release-cli-eval-support.mjs @@ -0,0 +1,325 @@ +import { execFileSync } from 'node:child_process'; +import { + chmodSync, + existsSync, + mkdirSync, + readFileSync, + readdirSync, + writeFileSync, +} from 'node:fs'; +import { basename, join, relative, sep } from 'node:path'; +import { pathToFileURL } from 'node:url'; + +const PROCESS_TIMEOUT_MS = 10 * 60 * 1000; +const MAX_OUTPUT_BYTES = 2 * 1024 * 1024; +const MAX_DIAGNOSTIC_TEXT = 4_000; + +export function findEvalReleaseTarball( + releaseDirectory, + platform = process.platform, + arch = process.arch, +) { + if (platform !== 'linux' || arch !== 'x64') { + throw new Error('The real Eval release validation requires Linux x64'); + } + const tarballs = readdirSync(releaseDirectory) + .filter((name) => /^maka-agent-[^/]+\.tgz$/u.test(name)) + .map((name) => join(releaseDirectory, name)); + if (tarballs.length !== 1) { + throw new Error( + `Expected one release tarball in ${releaseDirectory}, found ${tarballs.length}`, + ); + } + return tarballs[0]; +} + +export function isolatedEnvironment(home) { + mkdirSync(home, { recursive: true, mode: 0o700 }); + const environment = { + ...process.env, + HOME: home, + NODE_PATH: '', + XDG_CACHE_HOME: join(home, '.cache'), + XDG_CONFIG_HOME: join(home, '.config'), + XDG_DATA_HOME: join(home, '.local/share'), + }; + delete environment.PYTHONHOME; + delete environment.PYTHONPATH; + for (const name of Object.keys(environment)) { + if (name.startsWith('MAKA_EVAL_') || name.startsWith('GIT_')) delete environment[name]; + } + environment.GIT_CONFIG_NOSYSTEM = '1'; + for (const name of [ + 'ANTHROPIC_API_KEY', + 'DEEPSEEK_API_KEY', + 'OPENAI_API_KEY', + 'OPENROUTER_API_KEY', + ]) { + delete environment[name]; + } + return environment; +} + +export function createTaskFixture(root, environment, taskImage) { + const repositoryRoot = join(root, 'repository'); + const taskRoot = join(repositoryRoot, 'task'); + mkdirSync(join(taskRoot, 'environment'), { recursive: true, mode: 0o700 }); + mkdirSync(join(taskRoot, 'tests'), { recursive: true, mode: 0o700 }); + writeFileSync( + join(taskRoot, 'task.toml'), + [ + 'version = "1.0"', + '', + '[metadata]', + '', + '[verifier]', + 'timeout_sec = 60.0', + '', + '[agent]', + 'timeout_sec = 60.0', + '', + '[environment]', + 'build_timeout_sec = 120.0', + '', + ].join('\n'), + { mode: 0o600 }, + ); + writeFileSync( + join(taskRoot, 'instruction.md'), + 'Exit successfully without modifying the task.\n', + { mode: 0o600 }, + ); + writeFileSync(join(taskRoot, 'environment/Dockerfile'), `FROM ${taskImage}\nWORKDIR /app\n`, { + mode: 0o600, + }); + const testPath = join(taskRoot, 'tests/test.sh'); + writeFileSync(testPath, '#!/bin/sh\nset -eu\nprintf "1\\n" > /logs/verifier/reward.txt\n', { + mode: 0o700, + }); + chmodSync(testPath, 0o700); + run('git', ['init', '--initial-branch=main'], environment, repositoryRoot); + run('git', ['config', 'user.name', 'Maka Release Validation'], environment, repositoryRoot); + run( + 'git', + ['config', 'user.email', 'release-validation@maka.invalid'], + environment, + repositoryRoot, + ); + run('git', ['add', '.'], environment, repositoryRoot); + run( + 'git', + ['commit', '--message', 'Add deterministic release validation task'], + environment, + repositoryRoot, + ); + const commit = run('git', ['rev-parse', 'HEAD'], environment, repositoryRoot).trim(); + return { + commit, + repository: pathToFileURL(repositoryRoot).href, + tasksRoot: repositoryRoot, + }; +} + +export function readFrameworkOutputs({ framework, invocation, outputRoot, trialsRoot }) { + if (invocation.status !== 0) { + throw frameworkOutputError( + framework, + `exited with status ${invocation.status}`, + invocation, + [], + findJsonDiagnostics(trialsRoot), + ); + } + const attempts = findAttemptFiles(join(outputRoot, 'attempts')); + const attemptNames = attempts.map((path) => relative(outputRoot, path).split(sep).join('/')); + if (attempts.length !== 1) { + throw frameworkOutputError( + framework, + `produced ${attempts.length} attempt files instead of one`, + invocation, + attemptNames, + findJsonDiagnostics(trialsRoot), + ); + } + const summary = decodeJsonRecord(invocation.stdout); + if (!summary.ok) { + throw frameworkOutputError( + framework, + 'produced invalid summary JSON', + invocation, + attemptNames, + findJsonDiagnostics(trialsRoot), + { summary: summary.evidence }, + ); + } + const attempt = decodeJsonRecord(readFileSync(attempts[0], 'utf8')); + if (!attempt.ok) { + throw frameworkOutputError( + framework, + 'produced invalid attempt JSON', + invocation, + attemptNames, + findJsonDiagnostics(trialsRoot), + { attempt: attempt.evidence }, + ); + } + return { summary: summary.value, attempt: attempt.value }; +} + +export function findJsonDiagnostics(root, current = root, diagnostics = {}) { + if (!existsSync(current)) return diagnostics; + let entries; + try { + entries = readdirSync(current, { withFileTypes: true }); + } catch (error) { + const name = relative(root, current).split(sep).join('/') || '.'; + diagnostics[name] = diagnosticIoError('list', error); + return diagnostics; + } + for (const entry of entries) { + const path = join(current, entry.name); + if (entry.isDirectory()) { + findJsonDiagnostics(root, path, diagnostics); + continue; + } + if (!['preparation-error.json', 'result.json', 'trial.log'].includes(entry.name)) continue; + const name = relative(root, path).split(sep).join('/'); + let content; + try { + content = readFileSync(path, 'utf8').trim(); + } catch (error) { + diagnostics[name] = diagnosticIoError('read', error); + continue; + } + if (entry.name === 'trial.log') { + diagnostics[name] = tail(content); + continue; + } + const decoded = decodeJson(content); + if (!decoded.ok) { + diagnostics[name] = decoded.evidence; + continue; + } + const parsed = decoded.value; + diagnostics[name] = + entry.name === 'result.json' && isRecord(parsed) + ? { + exceptionInfo: isRecord(parsed.exception_info) + ? { + type: parsed.exception_info.exception_type, + message: tail(String(parsed.exception_info.exception_message ?? '')), + } + : null, + verifierResult: parsed.verifier_result ?? null, + } + : parsed; + } + return diagnostics; +} + +export function run(command, args, environment, cwd) { + try { + return execFileSync(command, args, { + cwd, + env: environment, + encoding: 'utf8', + timeout: PROCESS_TIMEOUT_MS, + maxBuffer: MAX_OUTPUT_BYTES, + stdio: ['ignore', 'pipe', 'pipe'], + }); + } catch (error) { + const stdout = String(error.stdout ?? '').trim(); + const stderr = String(error.stderr ?? '').trim(); + throw new Error( + `${command} ${args.join(' ')} failed${stdout ? `\nstdout:\n${stdout}` : ''}${stderr ? `\nstderr:\n${stderr}` : ''}`, + { cause: error }, + ); + } +} + +export function runAllowingFailure(command, args, environment, cwd) { + try { + return { status: 0, stdout: run(command, args, environment, cwd), stderr: '' }; + } catch (error) { + const cause = error.cause; + if (typeof cause?.status !== 'number') throw error; + return { + status: cause.status, + stdout: String(cause.stdout ?? ''), + stderr: String(cause.stderr ?? ''), + }; + } +} + +function findAttemptFiles(root) { + if (!existsSync(root)) return []; + const files = []; + for (const entry of readdirSync(root, { withFileTypes: true })) { + const path = join(root, entry.name); + if (entry.isDirectory()) files.push(...findAttemptFiles(path)); + else if (/^\d{6}\.json$/u.test(entry.name)) files.push(path); + } + return files.sort(); +} + +function frameworkOutputError( + framework, + reason, + invocation, + attempts, + frameworkDiagnostics, + extra = {}, +) { + return new Error( + `${framework} ${reason}: ${JSON.stringify({ + invocation: { + status: invocation.status, + stdout: tail(invocation.stdout), + stderr: tail(invocation.stderr), + }, + attempts, + ...extra, + frameworkDiagnostics, + })}`, + ); +} + +function decodeJsonRecord(content) { + const decoded = decodeJson(content); + if (!decoded.ok) return decoded; + if (!isRecord(decoded.value)) { + return { + ok: false, + evidence: { parseError: 'expected a JSON object', raw: tail(String(content).trim()) }, + }; + } + return decoded; +} + +function decodeJson(content) { + const raw = String(content).trim(); + try { + return { ok: true, value: JSON.parse(raw) }; + } catch (error) { + return { + ok: false, + evidence: { + parseError: error instanceof Error ? error.message : String(error), + raw: tail(raw), + }, + }; + } +} + +function isRecord(value) { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +function diagnosticIoError(operation, error) { + const code = typeof error?.code === 'string' ? error.code : 'UNKNOWN'; + return { diagnosticError: { operation, code } }; +} + +function tail(value) { + return String(value).slice(-MAX_DIAGNOSTIC_TEXT); +} diff --git a/scripts/release-cli-eval-support.test.mjs b/scripts/release-cli-eval-support.test.mjs new file mode 100644 index 0000000000..bd3ebc71cc --- /dev/null +++ b/scripts/release-cli-eval-support.test.mjs @@ -0,0 +1,131 @@ +import assert from 'node:assert/strict'; +import { mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, test } from 'node:test'; +import { + createTaskFixture, + findEvalReleaseTarball, + isolatedEnvironment, + readFrameworkOutputs, +} from './release-cli-eval-support.mjs'; + +describe('installed Eval release validation support', () => { + test('reports the unsupported platform before inspecting release artifacts', () => { + const root = mkdtempSync(join(tmpdir(), 'maka-eval-support-platform-')); + try { + assert.throws( + () => findEvalReleaseTarball(join(root, 'missing-release'), 'win32', 'x64'), + /requires Linux x64/u, + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + test('creates the Git fixture without inheriting host Git configuration', () => { + const root = mkdtempSync(join(tmpdir(), 'maka-eval-support-git-')); + const globalConfig = join(root, 'host.gitconfig'); + writeFileSync(globalConfig, '[commit]\n\tgpgSign = true\n', 'utf8'); + const previous = process.env.GIT_CONFIG_GLOBAL; + process.env.GIT_CONFIG_GLOBAL = globalConfig; + try { + const environment = isolatedEnvironment(join(root, 'home')); + assert.equal(environment.GIT_CONFIG_GLOBAL, undefined); + const fixture = createTaskFixture( + join(root, 'fixture'), + environment, + 'python:3.12-slim@sha256:test', + ); + assert.match(fixture.commit, /^[0-9a-f]{40}$/u); + } finally { + if (previous === undefined) delete process.env.GIT_CONFIG_GLOBAL; + else process.env.GIT_CONFIG_GLOBAL = previous; + rmSync(root, { recursive: true, force: true }); + } + }); + + test('preserves the primary process failure when diagnostics are truncated', () => { + const root = mkdtempSync(join(tmpdir(), 'maka-eval-support-failure-')); + const trialsRoot = join(root, 'trials'); + mkdirSync(join(trialsRoot, 'trial'), { recursive: true }); + writeFileSync(join(trialsRoot, 'trial/preparation-error.json'), '{"broken":', 'utf8'); + try { + assert.throws( + () => + readFrameworkOutputs({ + framework: 'pier', + invocation: { status: 1, stdout: '', stderr: 'primary framework failure' }, + outputRoot: join(root, 'output'), + trialsRoot, + }), + (error) => { + assert.match(error.message, /pier exited with status 1/u); + assert.match(error.message, /primary framework failure/u); + assert.match(error.message, /parseError/u); + return true; + }, + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + test('preserves the primary process failure when diagnostics cannot be read', { + skip: process.platform === 'win32', + }, () => { + const root = mkdtempSync(join(tmpdir(), 'maka-eval-support-unreadable-')); + const trialsRoot = join(root, 'trials'); + mkdirSync(join(trialsRoot, 'trial'), { recursive: true }); + symlinkSync('missing-trial.log', join(trialsRoot, 'trial/trial.log')); + try { + assert.throws( + () => + readFrameworkOutputs({ + framework: 'harbor', + invocation: { status: 1, stdout: '', stderr: 'primary framework failure' }, + outputRoot: join(root, 'output'), + trialsRoot, + }), + (error) => { + assert.match(error.message, /harbor exited with status 1/u); + assert.match(error.message, /primary framework failure/u); + assert.match(error.message, /diagnosticError.*ENOENT/u); + return true; + }, + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + test('reports malformed successful output with the attempt evidence', () => { + const root = mkdtempSync(join(tmpdir(), 'maka-eval-support-output-')); + const attemptsRoot = join(root, 'output/attempts/cell'); + mkdirSync(attemptsRoot, { recursive: true }); + writeFileSync( + join(attemptsRoot, '000001.json'), + JSON.stringify({ cellId: 'task::1::subject', sequence: 1, result: {} }), + 'utf8', + ); + try { + assert.throws( + () => + readFrameworkOutputs({ + framework: 'harbor', + invocation: { status: 0, stdout: 'not-json', stderr: '' }, + outputRoot: join(root, 'output'), + trialsRoot: join(root, 'trials'), + }), + (error) => { + assert.match(error.message, /harbor produced invalid summary JSON/u); + assert.match(error.message, /parseError/u); + assert.match(error.message, /000001\.json/u); + return true; + }, + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); From 10a38491b7aa9c98aeba0644e4e4c764c8b46017 Mon Sep 17 00:00:00 2001 From: M4n5ter Date: Tue, 18 Aug 2026 13:55:01 +0800 Subject: [PATCH 3/3] test(eval): assert reserved mount errors exactly Use an exact predicate for the observable Pier mount rejection so path punctuation cannot weaken the regression check. --- packages/eval/src/__tests__/lifecycle-boundaries.test.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/eval/src/__tests__/lifecycle-boundaries.test.ts b/packages/eval/src/__tests__/lifecycle-boundaries.test.ts index d049fa1015..e1257a6521 100644 --- a/packages/eval/src/__tests__/lifecycle-boundaries.test.ts +++ b/packages/eval/src/__tests__/lifecycle-boundaries.test.ts @@ -1315,7 +1315,9 @@ test('Pier rejects configured mounts that collide with framework log ownership', }, 'experiment.json', ), - new RegExp(`Pier mount target ${target.replaceAll('/', '\\/')} is reserved`, 'u'), + (error) => + error instanceof Error && + error.message === `Pier mount target ${target} is reserved for framework logs`, ); } } finally {