diff --git a/.github/workflows/cli-package-validation.yml b/.github/workflows/cli-package-validation.yml index 40444feb72..60904b03e8 100644 --- a/.github/workflows/cli-package-validation.yml +++ b/.github/workflows/cli-package-validation.yml @@ -144,3 +144,36 @@ jobs: path: packages/cli/release - name: Validate the installed tarball run: node scripts/smoke-release-cli-package.mjs + + eval: + name: Validate installed CLI Eval + needs: build + runs-on: ubuntu-24.04 + timeout-minutes: 30 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: '24' + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.12' + - name: Select the release npm toolchain + run: npm install --global --no-audit --no-fund npm@11.12.1 + - name: Install pinned Eval frameworks + run: | + python -m venv "$RUNNER_TEMP/maka-harbor" + "$RUNNER_TEMP/maka-harbor/bin/python" -m pip install --disable-pip-version-check 'harbor==0.20.0' + python -m venv "$RUNNER_TEMP/maka-pier" + "$RUNNER_TEMP/maka-pier/bin/python" -m pip install --disable-pip-version-check 'datacurve-pier==0.3.0' + echo "MAKA_RELEASE_HARBOR_PYTHON=$RUNNER_TEMP/maka-harbor/bin/python" >> "$GITHUB_ENV" + echo "MAKA_RELEASE_PIER_PYTHON=$RUNNER_TEMP/maka-pier/bin/python" >> "$GITHUB_ENV" + - name: Download the release candidate + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: cli-release-candidate + path: packages/cli/release + - name: Validate real Harbor and Pier cells + run: npm run release:cli:eval diff --git a/package.json b/package.json index 0364a23e80..e9dd8f6d26 100644 --- a/package.json +++ b/package.json @@ -44,9 +44,10 @@ "check:cli-third-party-notices": "node scripts/generate-third-party-notices.mjs --target cli --check", "release:cli:pack": "node scripts/release-cli-package.mjs", "release:cli:smoke": "node scripts/smoke-release-cli-package.mjs", + "release:cli:eval": "node scripts/release-cli-eval-package.mjs", "generate:windows-cargo-notices": "node scripts/generate-windows-cargo-notices.mjs", "check:windows-cargo-notices": "node scripts/generate-windows-cargo-notices.mjs --check", - "check:release": "npm run check:stale && npm run check:third-party-notices && npm run check:cli-third-party-notices && node --test scripts/release-cli-file-policy.test.mjs scripts/release-cli-artifact-policy.test.mjs", + "check:release": "npm run check:stale && npm run check:third-party-notices && npm run check:cli-third-party-notices && node --test scripts/release-cli-file-policy.test.mjs scripts/release-cli-artifact-policy.test.mjs scripts/release-cli-eval-support.test.mjs", "package:macos-arm64": "node scripts/package-macos-arm64.mjs", "verify:macos-arm64": "node scripts/verify-macos-arm64-dmg.mjs", "package:windows-x64": "node scripts/package-windows-x64.mjs", diff --git a/packages/eval/src/__tests__/lifecycle-boundaries.test.ts b/packages/eval/src/__tests__/lifecycle-boundaries.test.ts index d612758266..e1257a6521 100644 --- a/packages/eval/src/__tests__/lifecycle-boundaries.test.ts +++ b/packages/eval/src/__tests__/lifecycle-boundaries.test.ts @@ -1295,7 +1295,37 @@ test('pier cannot declare an egress proxy it never enforces', () => { ); }); -test('launched trial environment does not inherit MAKA_EVAL_FRAMEWORK', { +test('Pier rejects configured mounts that collide with framework log ownership', () => { + const root = join(tmpdir(), 'maka-test-pier-reserved-mount'); + const restoreEnvironment = setEnvironment({ + MAKA_TEST_MOUNT: join(root, 'mount'), + MAKA_TEST_PYTHON: join(root, 'python'), + MAKA_TEST_TASKS: join(root, 'tasks'), + MAKA_TEST_TRIALS: join(root, 'trials'), + }); + try { + for (const target of ['/logs/agent/../agent', '/logs/verifier/reward.txt']) { + assert.throws( + () => + createPierExecutor( + { + ...executorConfig(), + tasksRootEnv: 'MAKA_TEST_TASKS', + mounts: [{ sourceEnv: 'MAKA_TEST_MOUNT', target, readOnly: true }], + }, + 'experiment.json', + ), + (error) => + error instanceof Error && + error.message === `Pier mount target ${target} is reserved for framework logs`, + ); + } + } finally { + restoreEnvironment(); + } +}); + +test('Pier preserves its log mounts without inheriting MAKA_EVAL_FRAMEWORK', { timeout: 10_000, }, async () => { const root = await mkdtemp(join(tmpdir(), 'maka-eval-framework-env-')); @@ -1309,6 +1339,8 @@ import { mkdir, readFile, writeFile } from 'node:fs/promises'; const config = JSON.parse(await readFile(process.argv.at(-1), 'utf8')); await writeFile(process.env.MAKA_TEST_ENV, JSON.stringify({ framework: process.env.MAKA_EVAL_FRAMEWORK ?? null, + mounts: config.environment.mounts, + trialName: config.trial_name, })); const socket = connect(config.agent.kwargs.relay_port, config.agent.kwargs.relay_host); socket.setEncoding('utf8'); @@ -1344,14 +1376,33 @@ socket.end(); MAKA_TEST_PYTHON: executable, MAKA_TEST_TRIALS: root, MAKA_TEST_ENV: envDump, + MAKA_TEST_MOUNT: root, + MAKA_TEST_TASKS: root, MAKA_EVAL_FRAMEWORK: 'pier', }); try { + const spec: ExperimentSpec = { + ...experiment(), + executor: { + kind: 'pier', + config: { + ...executorConfig(), + tasksRootEnv: 'MAKA_TEST_TASKS', + mounts: [{ sourceEnv: 'MAKA_TEST_MOUNT', target: '/input', readOnly: true }], + }, + }, + tasks: [{ id: 'task', input: 'solve', config: { pier: { path: 'task' } } }], + }; const results = await runExperiment({ - spec: experiment(), + spec, store: new FileAttemptStore(join(root, 'attempts')), - executor: createHarborExecutor( - { ...executorConfig(), preparationEnvironment: ['MAKA_TEST_ENV'] }, + executor: createPierExecutor( + { + ...executorConfig(), + tasksRootEnv: 'MAKA_TEST_TASKS', + preparationEnvironment: ['MAKA_TEST_ENV'], + mounts: [{ sourceEnv: 'MAKA_TEST_MOUNT', target: '/input', readOnly: true }], + }, join(root, 'experiment.json'), ), subjects: [ @@ -1372,7 +1423,26 @@ socket.end(); ], }); assert.equal(results.get('task::1::external')?.result.status, 'completed'); - assert.deepEqual(JSON.parse(await readFile(envDump, 'utf8')), { framework: null }); + const launched = JSON.parse(await readFile(envDump, 'utf8')) as { + framework: string | null; + mounts: Array<{ source: string; target: string }>; + trialName: string; + }; + assert.equal(launched.framework, null); + assert.deepEqual(launched.mounts, [ + { type: 'bind', source: root, target: '/input', read_only: true }, + { type: 'bind', source: join(root, launched.trialName, 'agent'), target: '/logs/agent' }, + { + type: 'bind', + source: join(root, launched.trialName, 'verifier'), + target: '/logs/verifier', + }, + { + type: 'bind', + source: join(root, launched.trialName, 'artifacts'), + target: '/logs/artifacts', + }, + ]); } finally { restoreEnvironment(); await rm(root, { recursive: true, force: true }); diff --git a/packages/eval/src/harness-executor.ts b/packages/eval/src/harness-executor.ts index dbdee59f66..99e71c5344 100644 --- a/packages/eval/src/harness-executor.ts +++ b/packages/eval/src/harness-executor.ts @@ -4,7 +4,7 @@ import { once } from 'node:events'; import { createReadStream } from 'node:fs'; import { chmod, lstat, mkdir, readFile, readdir, unlink, writeFile } from 'node:fs/promises'; import { createServer, type Server, type Socket } from 'node:net'; -import { basename, dirname, join, relative, resolve, sep } from 'node:path'; +import { basename, dirname, join, posix, relative, resolve, sep } from 'node:path'; import { createInterface } from 'node:readline'; import { decodeJsonObject, type ExperimentCell, type JsonObject } from './experiment.js'; import { @@ -33,6 +33,12 @@ import type { EvalResult } from './result.js'; export type HarnessFramework = 'harbor' | 'pier'; type RelayTransportStage = 'ready' | 'execute' | 'receive' | 'decision'; +const PIER_FRAMEWORK_LOG_MOUNTS = Object.freeze([ + { directory: 'agent', target: '/logs/agent' }, + { directory: 'verifier', target: '/logs/verifier' }, + { directory: 'artifacts', target: '/logs/artifacts' }, +]); + interface RelayTransportFailure { readonly stage: RelayTransportStage; readonly category: @@ -195,7 +201,12 @@ async function runHarnessAttempt( : await waitForTrial(state.child, { phase: 'completion' }); finalizationEvidence = completed; if (!finalizationConfirmed(completed)) throw new Error('Trial did not finalize cleanly'); - const verification = await readVerification(state, cell, Boolean(options.egressProxy)); + const verification = await readVerification( + state, + cell, + framework, + Boolean(options.egressProxy), + ); verificationConfirmedBeforeCancellation = !hostCancellationObserved; return verification; }, @@ -381,7 +392,7 @@ async function startTrial( const task = decodeTask(framework, options, cell); const timeoutMultiplier = positive(cell.budget.timeoutMultiplier, 'budget.timeoutMultiplier'); const egressPaths = await resolveEgressPaths(options); - const environmentConfig = resolveEnvironmentConfig(options, egressPaths); + const environmentConfig = resolveEnvironmentConfig(options, egressPaths, framework, trialPath); const networkPolicyPath = egressPaths?.networkPolicyPath; const executionEnvironment = { ...UNATTENDED_EXECUTION_ENVIRONMENT, @@ -751,6 +762,7 @@ function inspectEgressAudit(audit: Buffer): { async function readVerification( state: RelayState, cell: ExperimentCell, + framework: HarnessFramework, expectEgressAudit: boolean, ): Promise { const result = JSON.parse(await readFile(join(state.trialPath, 'result.json'), 'utf8')) as { @@ -777,7 +789,7 @@ async function readVerification( failureReason: `failed to read egress audit log ${egressAuditPath}${code ? ` (${code})` : ''}`, artifacts: [ { kind: 'trial', framework: cell.executor.kind, trialName: state.trialName }, - ...(await collectedArtifactInventory(state.trialPath)), + ...(await collectedArtifactInventory(state.trialPath, framework)), { kind: 'egress-audit-unreadable', path: EGRESS_AUDIT_ARTIFACT_PATH }, ], }; @@ -796,14 +808,20 @@ async function readVerification( failureReason: audit.failureReason ?? (score === null ? 'verifier produced no reward' : null), artifacts: [ { kind: 'trial', framework: cell.executor.kind, trialName: state.trialName }, - ...(await collectedArtifactInventory(state.trialPath)), + ...(await collectedArtifactInventory(state.trialPath, framework)), ...audit.artifacts, ], }; } -async function collectedArtifactInventory(trialPath: string): Promise { - const root = join(trialPath, 'artifacts', 'logs', 'artifacts'); +async function collectedArtifactInventory( + trialPath: string, + framework: HarnessFramework, +): Promise { + const root = + framework === 'pier' + ? join(trialPath, 'artifacts') + : join(trialPath, 'artifacts', 'logs', 'artifacts'); const files: JsonObject[] = []; const targets = [ join(root, basename(MAKA_RUNTIME_ARTIFACT_PATH)), @@ -910,6 +928,18 @@ function decodeHarnessOptions(value: JsonObject, framework: HarnessFramework): H ? { tasksRootEnv: machinePathEnv(options.tasksRootEnv, 'tasksRootEnv') } : {}), }; + if (framework === 'pier') { + const reservedTargets = PIER_FRAMEWORK_LOG_MOUNTS.map((mount) => mount.target); + const collision = decoded.mounts.find((mount) => { + const target = posix.normalize(mount.target); + return reservedTargets.some( + (reserved) => target === reserved || target.startsWith(`${reserved}/`), + ); + }); + if (collision) { + throw new Error(`Pier mount target ${collision.target} is reserved for framework logs`); + } + } for (const name of [ decoded.pythonPathEnv, decoded.trialsRootEnv, @@ -980,8 +1010,22 @@ interface ResolvedEgressPaths { function resolveEnvironmentConfig( options: HarnessOptions, egressPaths: ResolvedEgressPaths | undefined, + framework: HarnessFramework, + trialPath: string, ): JsonObject { - const base = { ...options.environment, mounts: resolveMounts(options.mounts) }; + const configuredMounts = resolveMounts(options.mounts); + const mounts = + framework === 'pier' + ? [ + ...configuredMounts, + ...PIER_FRAMEWORK_LOG_MOUNTS.map(({ directory, target }) => ({ + type: 'bind', + source: join(trialPath, directory), + target, + })), + ] + : configuredMounts; + const base = { ...options.environment, mounts }; if (!options.egressProxy) return base; if (!egressPaths) throw new Error('egress proxy paths are unavailable'); return { ...base, extra_docker_compose: [egressPaths.composePath] }; diff --git a/scripts/release-cli-eval-package.mjs b/scripts/release-cli-eval-package.mjs new file mode 100644 index 0000000000..82d3f6b67c --- /dev/null +++ b/scripts/release-cli-eval-package.mjs @@ -0,0 +1,215 @@ +import { createHash } from 'node:crypto'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { basename, join, resolve } from 'node:path'; +import { + createTaskFixture, + findEvalReleaseTarball, + findJsonDiagnostics, + isolatedEnvironment, + readFrameworkOutputs, + run, + runAllowingFailure, +} from './release-cli-eval-support.mjs'; + +const HARBOR_VERSION = '0.20.0'; +const PIER_VERSION = '0.3.0'; +const TASK_IMAGE = + 'python:3.12-slim@sha256:dd29372629eeba2dd003fd9e9d35a5b8236c44727875a0364254b5127af88e65'; +const releaseDirectory = resolve('packages/cli/release'); +const tarballPath = findEvalReleaseTarball(releaseDirectory); + +const root = mkdtempSync(join(tmpdir(), 'maka-cli-eval-validation-')); +let primaryError; +try { + validateChecksum(); + const prefix = join(root, 'prefix'); + const environment = isolatedEnvironment(join(root, 'home')); + logStep('installing the immutable tarball with an empty offline npm cache'); + run( + 'npm', + [ + 'install', + '--global', + '--ignore-scripts', + '--offline', + '--no-audit', + '--no-fund', + '--cache', + join(root, 'npm-cache'), + '--prefix', + prefix, + tarballPath, + ], + environment, + root, + ); + + const maka = join(prefix, 'bin/maka'); + const packageRoot = join(prefix, 'lib/node_modules/maka-agent'); + if (!existsSync(maka) || !existsSync(join(packageRoot, 'packages/eval/harbor/run_trial.py'))) { + throw new Error('The installed candidate is missing its CLI or bundled Eval runtime'); + } + + const fixture = createTaskFixture(join(root, 'fixture'), environment, TASK_IMAGE); + for (const framework of ['harbor', 'pier']) { + logStep(`running a real ${framework} Docker cell from the installed candidate`); + validateFramework({ + environment, + fixture, + framework, + maka, + root: join(root, framework), + }); + } + logStep(`OK — installed ${basename(tarballPath)} completed real Harbor and Pier cells`); +} catch (error) { + primaryError = error; +} finally { + let cleanupError; + try { + rmSync(root, { recursive: true, force: true, maxRetries: 5, retryDelay: 200 }); + } catch (error) { + cleanupError = error; + } + if (primaryError && cleanupError) { + throw new AggregateError([primaryError, cleanupError], 'Eval validation and cleanup failed'); + } + if (primaryError) throw primaryError; + if (cleanupError) throw cleanupError; +} + +function validateFramework({ environment, fixture, framework, maka, root: frameworkRoot }) { + mkdirSync(frameworkRoot, { recursive: true, mode: 0o700 }); + const outputRoot = join(frameworkRoot, 'output'); + const trialsRoot = join(frameworkRoot, 'trials'); + const pythonVariable = + framework === 'harbor' ? 'MAKA_RELEASE_HARBOR_PYTHON' : 'MAKA_RELEASE_PIER_PYTHON'; + const pythonPath = process.env[pythonVariable]; + if (!pythonPath) throw new Error(`${pythonVariable} is required`); + const pythonPathVariable = `MAKA_RELEASE_${framework.toUpperCase()}_PYTHON_PATH`; + const trialsVariable = `MAKA_RELEASE_${framework.toUpperCase()}_TRIALS`; + const tasksVariable = 'MAKA_RELEASE_PIER_TASKS'; + const specPath = join(frameworkRoot, 'experiment.json'); + const spec = experimentSpec({ + fixture, + framework, + pythonPathVariable, + tasksVariable, + trialsVariable, + }); + writeFileSync(specPath, `${JSON.stringify(spec)}\n`, { mode: 0o600 }); + const childEnvironment = { + ...environment, + [pythonPathVariable]: pythonPath, + [trialsVariable]: trialsRoot, + ...(framework === 'pier' ? { [tasksVariable]: fixture.tasksRoot } : {}), + }; + const invocation = runAllowingFailure( + maka, + ['eval', 'run', specPath, '--out', outputRoot], + childEnvironment, + frameworkRoot, + ); + const { summary, attempt } = readFrameworkOutputs({ + framework, + invocation, + outputRoot, + trialsRoot, + }); + const result = attempt.result; + if ( + summary.experimentId !== `release-${framework}` || + summary.cells !== 1 || + summary.incomplete !== 0 || + attempt.cellId !== 'task::1::subject' || + attempt.sequence !== 1 || + result?.status !== 'completed' || + result?.score !== 1 || + result?.usage !== null || + result?.costUsd !== null || + result?.failureReason !== null + ) { + throw new Error( + `${framework} produced an invalid completed attempt: ${JSON.stringify({ + invocation, + summary, + attempt, + frameworkDiagnostics: findJsonDiagnostics(trialsRoot), + })}`, + ); + } + const trialArtifact = result.artifacts.find( + (artifact) => artifact.kind === 'trial' && artifact.framework === framework, + ); + const processArtifact = result.artifacts.find( + (artifact) => artifact.kind === 'external_process' && artifact.exitCode === 0, + ); + const collected = result.artifacts.filter((artifact) => artifact.kind === 'collected-artifact'); + if (!trialArtifact || !processArtifact || collected.length < 2) { + throw new Error(`${framework} did not preserve the expected trial and subject artifacts`); + } + const containers = run( + 'docker', + ['ps', '--all', '--quiet', '--filter', `name=${trialArtifact.trialName}`], + childEnvironment, + frameworkRoot, + ).trim(); + if (containers) throw new Error(`${framework} left trial containers behind: ${containers}`); +} + +function experimentSpec({ fixture, framework, pythonPathVariable, tasksVariable, trialsVariable }) { + return { + schemaVersion: 'maka.eval.v1', + id: `release-${framework}`, + benchmark: { + id: 'release-validation', + version: fixture.commit, + config: { repository: fixture.repository }, + }, + executor: { + kind: framework, + config: { + frameworkVersion: framework === 'harbor' ? HARBOR_VERSION : PIER_VERSION, + pythonPathEnv: pythonPathVariable, + trialsRootEnv: trialsVariable, + ...(framework === 'pier' ? { tasksRootEnv: tasksVariable } : {}), + environment: { type: 'docker', delete: true }, + preparationEnvironment: [], + mounts: [], + }, + }, + subjects: [ + { + id: 'subject', + kind: 'external', + credentials: [], + config: { command: '/bin/true', args: [], result: 'exit-code' }, + }, + ], + tasks: [ + { + id: 'task', + input: 'Exit successfully without modifying the task.', + config: framework === 'harbor' ? { harbor: { path: 'task' } } : { pier: { path: 'task' } }, + }, + ], + repetitions: 1, + budget: { timeoutMultiplier: 1 }, + verifier: { reward: 'reward' }, + }; +} + +function validateChecksum() { + const checksumPath = `${tarballPath}.sha256`; + const [expected, name, extra] = readFileSync(checksumPath, 'utf8').trim().split(/\s+/u); + if (!expected || name !== basename(tarballPath) || extra !== undefined) { + throw new Error(`Invalid checksum file ${checksumPath}`); + } + const actual = createHash('sha256').update(readFileSync(tarballPath)).digest('hex'); + if (actual !== expected) throw new Error(`Checksum mismatch for ${tarballPath}`); +} + +function logStep(message) { + console.log(`[release-cli-eval-validation] ${message}`); +} diff --git a/scripts/release-cli-eval-support.mjs b/scripts/release-cli-eval-support.mjs new file mode 100644 index 0000000000..57e50a4f4d --- /dev/null +++ b/scripts/release-cli-eval-support.mjs @@ -0,0 +1,325 @@ +import { execFileSync } from 'node:child_process'; +import { + chmodSync, + existsSync, + mkdirSync, + readFileSync, + readdirSync, + writeFileSync, +} from 'node:fs'; +import { basename, join, relative, sep } from 'node:path'; +import { pathToFileURL } from 'node:url'; + +const PROCESS_TIMEOUT_MS = 10 * 60 * 1000; +const MAX_OUTPUT_BYTES = 2 * 1024 * 1024; +const MAX_DIAGNOSTIC_TEXT = 4_000; + +export function findEvalReleaseTarball( + releaseDirectory, + platform = process.platform, + arch = process.arch, +) { + if (platform !== 'linux' || arch !== 'x64') { + throw new Error('The real Eval release validation requires Linux x64'); + } + const tarballs = readdirSync(releaseDirectory) + .filter((name) => /^maka-agent-[^/]+\.tgz$/u.test(name)) + .map((name) => join(releaseDirectory, name)); + if (tarballs.length !== 1) { + throw new Error( + `Expected one release tarball in ${releaseDirectory}, found ${tarballs.length}`, + ); + } + return tarballs[0]; +} + +export function isolatedEnvironment(home) { + mkdirSync(home, { recursive: true, mode: 0o700 }); + const environment = { + ...process.env, + HOME: home, + NODE_PATH: '', + XDG_CACHE_HOME: join(home, '.cache'), + XDG_CONFIG_HOME: join(home, '.config'), + XDG_DATA_HOME: join(home, '.local/share'), + }; + delete environment.PYTHONHOME; + delete environment.PYTHONPATH; + for (const name of Object.keys(environment)) { + if (name.startsWith('MAKA_EVAL_') || name.startsWith('GIT_')) delete environment[name]; + } + environment.GIT_CONFIG_NOSYSTEM = '1'; + for (const name of [ + 'ANTHROPIC_API_KEY', + 'DEEPSEEK_API_KEY', + 'OPENAI_API_KEY', + 'OPENROUTER_API_KEY', + ]) { + delete environment[name]; + } + return environment; +} + +export function createTaskFixture(root, environment, taskImage) { + const repositoryRoot = join(root, 'repository'); + const taskRoot = join(repositoryRoot, 'task'); + mkdirSync(join(taskRoot, 'environment'), { recursive: true, mode: 0o700 }); + mkdirSync(join(taskRoot, 'tests'), { recursive: true, mode: 0o700 }); + writeFileSync( + join(taskRoot, 'task.toml'), + [ + 'version = "1.0"', + '', + '[metadata]', + '', + '[verifier]', + 'timeout_sec = 60.0', + '', + '[agent]', + 'timeout_sec = 60.0', + '', + '[environment]', + 'build_timeout_sec = 120.0', + '', + ].join('\n'), + { mode: 0o600 }, + ); + writeFileSync( + join(taskRoot, 'instruction.md'), + 'Exit successfully without modifying the task.\n', + { mode: 0o600 }, + ); + writeFileSync(join(taskRoot, 'environment/Dockerfile'), `FROM ${taskImage}\nWORKDIR /app\n`, { + mode: 0o600, + }); + const testPath = join(taskRoot, 'tests/test.sh'); + writeFileSync(testPath, '#!/bin/sh\nset -eu\nprintf "1\\n" > /logs/verifier/reward.txt\n', { + mode: 0o700, + }); + chmodSync(testPath, 0o700); + run('git', ['init', '--initial-branch=main'], environment, repositoryRoot); + run('git', ['config', 'user.name', 'Maka Release Validation'], environment, repositoryRoot); + run( + 'git', + ['config', 'user.email', 'release-validation@maka.invalid'], + environment, + repositoryRoot, + ); + run('git', ['add', '.'], environment, repositoryRoot); + run( + 'git', + ['commit', '--message', 'Add deterministic release validation task'], + environment, + repositoryRoot, + ); + const commit = run('git', ['rev-parse', 'HEAD'], environment, repositoryRoot).trim(); + return { + commit, + repository: pathToFileURL(repositoryRoot).href, + tasksRoot: repositoryRoot, + }; +} + +export function readFrameworkOutputs({ framework, invocation, outputRoot, trialsRoot }) { + if (invocation.status !== 0) { + throw frameworkOutputError( + framework, + `exited with status ${invocation.status}`, + invocation, + [], + findJsonDiagnostics(trialsRoot), + ); + } + const attempts = findAttemptFiles(join(outputRoot, 'attempts')); + const attemptNames = attempts.map((path) => relative(outputRoot, path).split(sep).join('/')); + if (attempts.length !== 1) { + throw frameworkOutputError( + framework, + `produced ${attempts.length} attempt files instead of one`, + invocation, + attemptNames, + findJsonDiagnostics(trialsRoot), + ); + } + const summary = decodeJsonRecord(invocation.stdout); + if (!summary.ok) { + throw frameworkOutputError( + framework, + 'produced invalid summary JSON', + invocation, + attemptNames, + findJsonDiagnostics(trialsRoot), + { summary: summary.evidence }, + ); + } + const attempt = decodeJsonRecord(readFileSync(attempts[0], 'utf8')); + if (!attempt.ok) { + throw frameworkOutputError( + framework, + 'produced invalid attempt JSON', + invocation, + attemptNames, + findJsonDiagnostics(trialsRoot), + { attempt: attempt.evidence }, + ); + } + return { summary: summary.value, attempt: attempt.value }; +} + +export function findJsonDiagnostics(root, current = root, diagnostics = {}) { + if (!existsSync(current)) return diagnostics; + let entries; + try { + entries = readdirSync(current, { withFileTypes: true }); + } catch (error) { + const name = relative(root, current).split(sep).join('/') || '.'; + diagnostics[name] = diagnosticIoError('list', error); + return diagnostics; + } + for (const entry of entries) { + const path = join(current, entry.name); + if (entry.isDirectory()) { + findJsonDiagnostics(root, path, diagnostics); + continue; + } + if (!['preparation-error.json', 'result.json', 'trial.log'].includes(entry.name)) continue; + const name = relative(root, path).split(sep).join('/'); + let content; + try { + content = readFileSync(path, 'utf8').trim(); + } catch (error) { + diagnostics[name] = diagnosticIoError('read', error); + continue; + } + if (entry.name === 'trial.log') { + diagnostics[name] = tail(content); + continue; + } + const decoded = decodeJson(content); + if (!decoded.ok) { + diagnostics[name] = decoded.evidence; + continue; + } + const parsed = decoded.value; + diagnostics[name] = + entry.name === 'result.json' && isRecord(parsed) + ? { + exceptionInfo: isRecord(parsed.exception_info) + ? { + type: parsed.exception_info.exception_type, + message: tail(String(parsed.exception_info.exception_message ?? '')), + } + : null, + verifierResult: parsed.verifier_result ?? null, + } + : parsed; + } + return diagnostics; +} + +export function run(command, args, environment, cwd) { + try { + return execFileSync(command, args, { + cwd, + env: environment, + encoding: 'utf8', + timeout: PROCESS_TIMEOUT_MS, + maxBuffer: MAX_OUTPUT_BYTES, + stdio: ['ignore', 'pipe', 'pipe'], + }); + } catch (error) { + const stdout = String(error.stdout ?? '').trim(); + const stderr = String(error.stderr ?? '').trim(); + throw new Error( + `${command} ${args.join(' ')} failed${stdout ? `\nstdout:\n${stdout}` : ''}${stderr ? `\nstderr:\n${stderr}` : ''}`, + { cause: error }, + ); + } +} + +export function runAllowingFailure(command, args, environment, cwd) { + try { + return { status: 0, stdout: run(command, args, environment, cwd), stderr: '' }; + } catch (error) { + const cause = error.cause; + if (typeof cause?.status !== 'number') throw error; + return { + status: cause.status, + stdout: String(cause.stdout ?? ''), + stderr: String(cause.stderr ?? ''), + }; + } +} + +function findAttemptFiles(root) { + if (!existsSync(root)) return []; + const files = []; + for (const entry of readdirSync(root, { withFileTypes: true })) { + const path = join(root, entry.name); + if (entry.isDirectory()) files.push(...findAttemptFiles(path)); + else if (/^\d{6}\.json$/u.test(entry.name)) files.push(path); + } + return files.sort(); +} + +function frameworkOutputError( + framework, + reason, + invocation, + attempts, + frameworkDiagnostics, + extra = {}, +) { + return new Error( + `${framework} ${reason}: ${JSON.stringify({ + invocation: { + status: invocation.status, + stdout: tail(invocation.stdout), + stderr: tail(invocation.stderr), + }, + attempts, + ...extra, + frameworkDiagnostics, + })}`, + ); +} + +function decodeJsonRecord(content) { + const decoded = decodeJson(content); + if (!decoded.ok) return decoded; + if (!isRecord(decoded.value)) { + return { + ok: false, + evidence: { parseError: 'expected a JSON object', raw: tail(String(content).trim()) }, + }; + } + return decoded; +} + +function decodeJson(content) { + const raw = String(content).trim(); + try { + return { ok: true, value: JSON.parse(raw) }; + } catch (error) { + return { + ok: false, + evidence: { + parseError: error instanceof Error ? error.message : String(error), + raw: tail(raw), + }, + }; + } +} + +function isRecord(value) { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +function diagnosticIoError(operation, error) { + const code = typeof error?.code === 'string' ? error.code : 'UNKNOWN'; + return { diagnosticError: { operation, code } }; +} + +function tail(value) { + return String(value).slice(-MAX_DIAGNOSTIC_TEXT); +} diff --git a/scripts/release-cli-eval-support.test.mjs b/scripts/release-cli-eval-support.test.mjs new file mode 100644 index 0000000000..bd3ebc71cc --- /dev/null +++ b/scripts/release-cli-eval-support.test.mjs @@ -0,0 +1,131 @@ +import assert from 'node:assert/strict'; +import { mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, test } from 'node:test'; +import { + createTaskFixture, + findEvalReleaseTarball, + isolatedEnvironment, + readFrameworkOutputs, +} from './release-cli-eval-support.mjs'; + +describe('installed Eval release validation support', () => { + test('reports the unsupported platform before inspecting release artifacts', () => { + const root = mkdtempSync(join(tmpdir(), 'maka-eval-support-platform-')); + try { + assert.throws( + () => findEvalReleaseTarball(join(root, 'missing-release'), 'win32', 'x64'), + /requires Linux x64/u, + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + test('creates the Git fixture without inheriting host Git configuration', () => { + const root = mkdtempSync(join(tmpdir(), 'maka-eval-support-git-')); + const globalConfig = join(root, 'host.gitconfig'); + writeFileSync(globalConfig, '[commit]\n\tgpgSign = true\n', 'utf8'); + const previous = process.env.GIT_CONFIG_GLOBAL; + process.env.GIT_CONFIG_GLOBAL = globalConfig; + try { + const environment = isolatedEnvironment(join(root, 'home')); + assert.equal(environment.GIT_CONFIG_GLOBAL, undefined); + const fixture = createTaskFixture( + join(root, 'fixture'), + environment, + 'python:3.12-slim@sha256:test', + ); + assert.match(fixture.commit, /^[0-9a-f]{40}$/u); + } finally { + if (previous === undefined) delete process.env.GIT_CONFIG_GLOBAL; + else process.env.GIT_CONFIG_GLOBAL = previous; + rmSync(root, { recursive: true, force: true }); + } + }); + + test('preserves the primary process failure when diagnostics are truncated', () => { + const root = mkdtempSync(join(tmpdir(), 'maka-eval-support-failure-')); + const trialsRoot = join(root, 'trials'); + mkdirSync(join(trialsRoot, 'trial'), { recursive: true }); + writeFileSync(join(trialsRoot, 'trial/preparation-error.json'), '{"broken":', 'utf8'); + try { + assert.throws( + () => + readFrameworkOutputs({ + framework: 'pier', + invocation: { status: 1, stdout: '', stderr: 'primary framework failure' }, + outputRoot: join(root, 'output'), + trialsRoot, + }), + (error) => { + assert.match(error.message, /pier exited with status 1/u); + assert.match(error.message, /primary framework failure/u); + assert.match(error.message, /parseError/u); + return true; + }, + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + test('preserves the primary process failure when diagnostics cannot be read', { + skip: process.platform === 'win32', + }, () => { + const root = mkdtempSync(join(tmpdir(), 'maka-eval-support-unreadable-')); + const trialsRoot = join(root, 'trials'); + mkdirSync(join(trialsRoot, 'trial'), { recursive: true }); + symlinkSync('missing-trial.log', join(trialsRoot, 'trial/trial.log')); + try { + assert.throws( + () => + readFrameworkOutputs({ + framework: 'harbor', + invocation: { status: 1, stdout: '', stderr: 'primary framework failure' }, + outputRoot: join(root, 'output'), + trialsRoot, + }), + (error) => { + assert.match(error.message, /harbor exited with status 1/u); + assert.match(error.message, /primary framework failure/u); + assert.match(error.message, /diagnosticError.*ENOENT/u); + return true; + }, + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + test('reports malformed successful output with the attempt evidence', () => { + const root = mkdtempSync(join(tmpdir(), 'maka-eval-support-output-')); + const attemptsRoot = join(root, 'output/attempts/cell'); + mkdirSync(attemptsRoot, { recursive: true }); + writeFileSync( + join(attemptsRoot, '000001.json'), + JSON.stringify({ cellId: 'task::1::subject', sequence: 1, result: {} }), + 'utf8', + ); + try { + assert.throws( + () => + readFrameworkOutputs({ + framework: 'harbor', + invocation: { status: 0, stdout: 'not-json', stderr: '' }, + outputRoot: join(root, 'output'), + trialsRoot: join(root, 'trials'), + }), + (error) => { + assert.match(error.message, /harbor produced invalid summary JSON/u); + assert.match(error.message, /parseError/u); + assert.match(error.message, /000001\.json/u); + return true; + }, + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/scripts/smoke-release-cli-package.mjs b/scripts/smoke-release-cli-package.mjs index 0037b721da..5d3d03112b 100644 --- a/scripts/smoke-release-cli-package.mjs +++ b/scripts/smoke-release-cli-package.mjs @@ -143,6 +143,9 @@ async function validateInstalledProduct(root) { ); validateInstalledRuntimeFiles(packageRoot); + logStep('checking installed Eval spec decoding and framework preflight'); + smokeEvalPreflight({ crossSpawn: crossSpawn.sync, environment: baseEnvironment, maka, root }); + logStep('checking installed native PTY and file-lock modules'); const nodePty = await importInstalled(packageRoot, 'node_modules/node-pty/lib/index.js'); const ptySpawn = nodePty.spawn ?? nodePty.default?.spawn; @@ -230,6 +233,78 @@ function validateInstalledRuntimeFiles(packageRoot) { ); } +function smokeEvalPreflight({ crossSpawn, environment, maka, root }) { + const evalRoot = join(root, 'eval-preflight'); + mkdirSync(evalRoot, { recursive: true }); + const machineEnvironment = { + ...environment, + MAKA_RELEASE_EVAL_PYTHON: process.execPath, + MAKA_RELEASE_EVAL_TASKS: evalRoot, + MAKA_RELEASE_EVAL_TRIALS: join(evalRoot, 'trials'), + }; + for (const framework of ['harbor', 'pier']) { + const specPath = join(evalRoot, `${framework}.json`); + writeFileSync(specPath, `${JSON.stringify(evalPreflightSpec(framework))}\n`, 'utf8'); + const result = crossSpawn(maka, ['eval', 'run', specPath, '--out', join(evalRoot, framework)], { + cwd: evalRoot, + env: machineEnvironment, + encoding: 'utf8', + timeout: 30_000, + maxBuffer: MAX_OUTPUT_BYTES, + }); + if (result.error) throw result.error; + if (result.status === 0) { + throw new Error(`${framework} Eval preflight unexpectedly accepted the Node executable`); + } + assertOutput( + `${result.stdout ?? ''}\n${result.stderr ?? ''}`, + `${framework} Python environment MAKA_RELEASE_EVAL_PYTHON is unavailable or does not provide`, + ); + } +} + +function evalPreflightSpec(framework) { + return { + schemaVersion: 'maka.eval.v1', + id: `release-${framework}-preflight`, + benchmark: { + id: 'release-preflight', + version: '0000000000000000000000000000000000000000', + config: { repository: 'https://invalid.invalid/release-preflight.git' }, + }, + executor: { + kind: framework, + config: { + frameworkVersion: framework === 'harbor' ? '0.20.0' : '0.3.0', + pythonPathEnv: 'MAKA_RELEASE_EVAL_PYTHON', + trialsRootEnv: 'MAKA_RELEASE_EVAL_TRIALS', + ...(framework === 'pier' ? { tasksRootEnv: 'MAKA_RELEASE_EVAL_TASKS' } : {}), + environment: { type: 'docker', delete: true }, + preparationEnvironment: [], + mounts: [], + }, + }, + subjects: [ + { + id: 'subject', + kind: 'external', + credentials: [], + config: { command: process.execPath, args: ['--version'], result: 'exit-code' }, + }, + ], + tasks: [ + { + id: 'task', + input: 'Do not execute this preflight-only task.', + config: framework === 'harbor' ? { harbor: { path: 'task' } } : { pier: { path: 'task' } }, + }, + ], + repetitions: 1, + budget: { timeoutMultiplier: 1 }, + verifier: { reward: 'reward' }, + }; +} + async function smokePty(ptySpawn, environment, cwd) { const result = await runPtyScenario({ ptySpawn,