From ad8f97901618d59a732570ed3de694ffe1d67fd2 Mon Sep 17 00:00:00 2001 From: kt Date: Wed, 30 Sep 2026 20:09:42 -0700 Subject: [PATCH 01/89] build darwin x64 on the m-pool (#3236) The x86_64-apple-darwin legs in continuous.yml and release.yml move from the macos-15-intel pool to the macos-15 Apple-silicon pool, cross-compiling for x86_64 (the target stays x86_64-apple-darwin). The M1 links this binary ~2.2x faster than the intel pool with far tighter variance (484 s vs 1046 s same-commit link tail), killing the 29-minute outliers; the livecheck runs the x64 artifact under Rosetta 2, the runtime install-rust.sh gives an x86_64 payload on an Apple-Silicon Mac. --- .github/workflows/continuous.yml | 19 +++++++++++++++++-- .github/workflows/release.yml | 15 ++++++++++++++- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/.github/workflows/continuous.yml b/.github/workflows/continuous.yml index dabe1976221..03572028373 100644 --- a/.github/workflows/continuous.yml +++ b/.github/workflows/continuous.yml @@ -234,8 +234,17 @@ jobs: include: - target: aarch64-apple-darwin runs-on: macos-14 + # The x86_64 leg cross-compiles on the Apple-silicon pool: rustc + # and the Xcode linker build either darwin arch from an M-series + # host, and the M-pool links this binary ~2.2x faster than the + # macos-15-intel pool with far tighter variance (484 s vs 1046 s + # same-commit link tail, 2026-09-30). The livecheck below runs + # the assembled x64 artifact under Rosetta 2 — the macos-15 + # arm64 image installs it, and install-rust.sh ships the x64 + # build to an Apple-Silicon Mac's Rosetta shell, so the gate + # still boots the binary on its delivery runtime. - target: x86_64-apple-darwin - runs-on: macos-15-intel # macos-13 is retired; this is the Intel runner now + runs-on: macos-15 env: # AppleDouble resource forks must not leak into the archive; the # assembler additionally enforces plain-file payloads. @@ -272,7 +281,13 @@ jobs: --catalog-assets "target/${{ matrix.target }}/catalog-assets" \ --out-dir "target/${{ matrix.target }}/dist" cat "target/${{ matrix.target }}/dist/manifest.json" - - name: Livecheck the assembled continuous artifact on its native runner + # The livecheck extracts the tarball and runs the payload binary's + # `--version` from a scratch cwd (verify_release.py). The aarch64 leg + # executes it natively; the x86_64 leg, cross-compiled on the + # Apple-silicon pool, executes it under Rosetta 2 — the runtime the + # x64 payload runs on for an Apple-Silicon Mac customer, so the gate + # still boots the artifact on its delivery runtime. + - name: Livecheck the assembled continuous artifact on its build runner run: | set -euo pipefail version="$(sed -n 's/^version *= *"\([^"]*\)".*/\1/p' Cargo.toml | head -1)" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 86bf9a05532..486bf6befca 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -234,8 +234,17 @@ jobs: include: - target: aarch64-apple-darwin runs-on: macos-14 + # The x86_64 leg cross-compiles on the Apple-silicon pool: rustc + # and the Xcode linker build either darwin arch from an M-series + # host, and the M-pool links this binary ~2.2x faster than the + # macos-15-intel pool with far tighter variance (484 s vs 1046 s + # same-commit link tail, 2026-09-30). The livecheck below runs + # the assembled x64 artifact under Rosetta 2 — the macos-15 + # arm64 image installs it, and install-rust.sh ships the x64 + # build to an Apple-Silicon Mac's Rosetta shell, so the gate + # still boots the binary on its delivery runtime. - target: x86_64-apple-darwin - runs-on: macos-15-intel # macos-13 is retired; this is the Intel runner now + runs-on: macos-15 env: # AppleDouble resource forks must not leak into the archive; the # assembler additionally enforces plain-file payloads. @@ -253,6 +262,10 @@ jobs: key: ${{ matrix.target }} - name: Build (release, locked) run: cargo build --release --locked --target ${{ matrix.target }} -j 4 + # The aarch64 leg livechecks natively; the x86_64 leg, + # cross-compiled on the Apple-silicon pool, livechecks under + # Rosetta 2 — the runtime install-rust.sh gives the x64 payload on + # an Apple-Silicon Mac. - name: "Livecheck gate: --version prints the tag version" run: | set -euo pipefail From 583b603119e2b21f719d4196b9b6b86efb396975 Mon Sep 17 00:00:00 2001 From: kt Date: Wed, 30 Sep 2026 20:09:59 -0700 Subject: [PATCH 02/89] remove the faux provider and the hillclimb perf tooling (#3223) The hillclimb era's measurement cluster dies whole: the TS-side faux-provider driver (scripts/ts_faux_extension.js) and the perf battery it drove - scripts/battery/ in its entirety (stream_throughput.py, streaming_render.py-era survivors, sustained_cpu.py, perf_wave.py, perf_gate.py, ci_perf_wave.sh, run_perf_wave.sh, the recorded perf-baseline.json, and the support libs that served nothing else: batterylib.py, mock_provider.py, perf.py, scale_corpus.py, ts_identity.py) - plus the Makefile perf-wave target and its .PHONY entry. Every member had zero live references: no workflow invocation (benchmark.yml is long gone, only the dead Makefile mirror remained), no test shells out to them, the crontab and the fleet tooling never touch them. The benchmark campaign harness on the bench node is the modern performance tooling. Folded onto main after the scripts purge (#3178) merged: that purge already removed the parity harnesses this lane's map had found importing the faux (acp_compaction_parity.py, compact_parity.py, mcp_hint_parity.py, paste_stream_parity.py, spinner_cadence.py, streaming_render.py and the visual-parity cluster), so those deletions fold to no-ops; the purge's merged form kept the wave serving only itself - the faux header named exactly the two wave dimensions as its consumers - so the whole battery directory goes here instead. The golden regeneration harness (crates/pa-core/tests/golden/harness.mjs) is not needed post-port either: the replay tests assert against the frozen corpus at runtime, and the TS checkout remains the reference if a golden ever needs re-derivation. The corpus stays, frozen provenance noted in the replay module doc. Doc/comment repairs where kept files named a removed script: the Makefile drops the perf-wave mirror, golden_replay.rs's harness.mjs cites become the corpus's provenance notes, and differential_cli.rs's TMPDIR comment no longer cites batterylib's scrubbed_env. --- Makefile | 10 +- crates/pa-cli/tests/differential_cli.rs | 2 +- crates/pa-core/src/tools/golden_replay.rs | 6 +- crates/pa-core/tests/golden/harness.mjs | 663 ------------- scripts/battery/README.md | 62 -- scripts/battery/batterylib.py | 715 -------------- scripts/battery/ci_perf_wave.sh | 106 --- scripts/battery/mock_provider.py | 297 ------ scripts/battery/perf-baseline.json | 235 ----- scripts/battery/perf.py | 250 ----- scripts/battery/perf_gate.py | 116 --- scripts/battery/perf_wave.py | 1026 --------------------- scripts/battery/run_perf_wave.sh | 46 - scripts/battery/scale_corpus.py | 183 ---- scripts/battery/stream_throughput.py | 258 ------ scripts/battery/sustained_cpu.py | 573 ------------ scripts/battery/ts_identity.py | 108 --- scripts/ts_faux_extension.js | 112 --- 18 files changed, 5 insertions(+), 4763 deletions(-) delete mode 100644 crates/pa-core/tests/golden/harness.mjs delete mode 100644 scripts/battery/README.md delete mode 100644 scripts/battery/batterylib.py delete mode 100755 scripts/battery/ci_perf_wave.sh delete mode 100644 scripts/battery/mock_provider.py delete mode 100644 scripts/battery/perf-baseline.json delete mode 100644 scripts/battery/perf.py delete mode 100644 scripts/battery/perf_gate.py delete mode 100644 scripts/battery/perf_wave.py delete mode 100755 scripts/battery/run_perf_wave.sh delete mode 100644 scripts/battery/scale_corpus.py delete mode 100755 scripts/battery/stream_throughput.py delete mode 100644 scripts/battery/sustained_cpu.py delete mode 100644 scripts/battery/ts_identity.py delete mode 100644 scripts/ts_faux_extension.js diff --git a/Makefile b/Makefile index 0b87aa7276f..0905902c962 100644 --- a/Makefile +++ b/Makefile @@ -51,14 +51,6 @@ glibc-gate: fi \ ;; esac -# Perf wave + regression gate (benchmark.yml job, the local mirror): runs the -# TS binary and a fresh release build side by side in a fresh Prime sandbox -# (both sides on one quiet machine, the methodology BENCHMARKS.md requires) -# and gates the rust medians against scripts/battery/perf-baseline.json. -# PA_BENCH_NO_SANDBOX=1 runs it on the bare runner instead. -perf-wave: - scripts/battery/ci_perf_wave.sh - # Local mirror of the release build-job gates: # release build against the committed lockfile, deterministic tarball assembly, # then end-to-end verification of the host-target artifact. The vendored @@ -180,4 +172,4 @@ shard-gates: python3 scripts/test_ci_test_shard.py python3 scripts/test_ci_pr_crates.py -.PHONY: check deny windows-cross actionlint perf-wave glibc-gate release-dry-run continuous-dry-run audit-build package catalog-assets catalog-assets-fixture catalog-assets-gates fold-gates restamp-gates shard-gates +.PHONY: check deny windows-cross actionlint glibc-gate release-dry-run continuous-dry-run audit-build package catalog-assets catalog-assets-fixture catalog-assets-gates fold-gates restamp-gates shard-gates diff --git a/crates/pa-cli/tests/differential_cli.rs b/crates/pa-cli/tests/differential_cli.rs index c5fc0024e78..c36e7a22f94 100644 --- a/crates/pa-cli/tests/differential_cli.rs +++ b/crates/pa-cli/tests/differential_cli.rs @@ -356,7 +356,7 @@ fn run(binary: &Path, args: &[&str], sandbox: &Path) -> InvocationOutput { // through passwd rather than $HOME, so the env override is the only // reliable isolation for both binaries. .env("PRIME_AGENT_CODING_AGENT_DIR", sandbox.join("agent")) - // Isolate TMPDIR too (batterylib.scrubbed_env parity): without it the + // Isolate TMPDIR too: without it the // TS daemon census escapes into the box's ambient daemons and // `shutdown --force` kills unrelated sockets (containment contract). .env("TMPDIR", sandbox.join("tmp")) diff --git a/crates/pa-core/src/tools/golden_replay.rs b/crates/pa-core/src/tools/golden_replay.rs index 307de2704ab..34d0b3c0cfd 100644 --- a/crates/pa-core/src/tools/golden_replay.rs +++ b/crates/pa-core/src/tools/golden_replay.rs @@ -1,5 +1,5 @@ //! Golden differential tests: replay the corpus recorded from the REAL -//! TypeScript tools (see `tests/golden/harness.mjs`) against the Rust port +//! TypeScript tools (frozen in `tests/golden/corpus/`) against the Rust port //! and assert identical model-facing output. //! //! Corpus cases cover fuzzy edits, file-not-found, bash timeouts and @@ -30,7 +30,7 @@ fn corpus(name: &str) -> serde_json::Value { serde_json::from_str(&std::fs::read_to_string(path).expect("corpus file")).expect("corpus JSON") } -/// Normalize temp paths the same way harness.mjs does. +/// Normalize temp paths to the forms the recorded corpus uses. fn norm_string(s: &str, tmp: &str) -> String { let out = s.replace(&format!("{tmp}/"), "/"); let re = regex_lite("golden-[A-Za-z0-9_]{6}"); @@ -127,7 +127,7 @@ async fn golden_edit_group_matches_ts() { let mut input = case["input"].clone(); if case["applyPrepareArguments"].as_bool().unwrap_or(false) { input = crate::tools::edit::prepare_edit_arguments(input); - // The TS harness records the prepared input for such cases. + // The TS harness recorded the prepared input for such cases. assert_json_eq( &input.clone(), &case["preparedInput"].clone(), diff --git a/crates/pa-core/tests/golden/harness.mjs b/crates/pa-core/tests/golden/harness.mjs deleted file mode 100644 index 07c940613fb..00000000000 --- a/crates/pa-core/tests/golden/harness.mjs +++ /dev/null @@ -1,663 +0,0 @@ -#!/usr/bin/env node -/** - * Golden differential harness (lane core-tools). - * - * Runs the REAL TypeScript tools from the /tmp/pa-golden copy of prime-agent - * and records their results as a JSON corpus, so a Rust port can assert - * byte-identical parity. - * - * - cwd for execution: /tmp/pa-golden (node_modules resolution via jiti). - * - Output: corpus/*.json next to this file. - * - Every case sets up its fixture in a fresh mkdtemp dir; the fixture setup - * is recorded as replayable shell commands. - * - Normalization: os.tmpdir() prefix -> "", random tmp ids -> "". - * No timestamps, durations, or random ids are recorded. - * - * Determinism: two runs produce identical corpus files. - */ -import { spawnSync } from "node:child_process"; -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; -import { fileURLToPath, pathToFileURL } from "node:url"; -import { createRequire } from "node:module"; - -const PA_ROOT = "/tmp/pa-golden"; -const __filename = fileURLToPath(import.meta.url); -const OUT_DIR = path.dirname(__filename); -const CORPUS = path.join(OUT_DIR, "corpus"); - -const TMP = os.tmpdir(); - -const req = createRequire(path.join(PA_ROOT, "package.json")); -const { createJiti } = req("jiti"); -const jiti = createJiti(pathToFileURL(path.join(PA_ROOT, "harness-entry.mjs")).href, { - interopDefault: true, - alias: { - "@earendil-works/pi-agent-core": path.join(PA_ROOT, "packages/agent/src/index.ts"), - "@earendil-works/pi-ai": path.join(PA_ROOT, "packages/ai/src/index.ts"), - "@earendil-works/pi-tui": path.join(PA_ROOT, "packages/tui/src/index.ts"), - }, -}); - -async function loadTs(rel) { - const mod = await jiti.import(path.join(PA_ROOT, rel)); - return mod.default ?? mod; -} - -// ---------------------------------------------------------------- normalizing -function normString(s) { - let out = s.split(TMP + "/").join("/"); - out = out.replace(/golden-[A-Za-z0-9_]{6}/g, "golden-"); - out = out.replace(/\bpi-(?:bash|output)-[0-9a-f]+\.log/g, "pi-.log"); - out = out.replace(/tmp-stdout-[0-9A-Za-z_]+/g, "tmp-stdout-"); - return out; -} - -function deepNorm(value) { - if (typeof value === "string") return normString(value); - if (Array.isArray(value)) return value.map(deepNorm); - if (value && typeof value === "object") { - const out = {}; - for (const [k, v] of Object.entries(value)) out[k] = deepNorm(v); - return out; - } - return value; -} - -function writeCorpus(name, cases) { - const doc = deepNorm({ group: name, caseCount: cases.length, cases }); - fs.writeFileSync(path.join(CORPUS, name + ".json"), JSON.stringify(doc, null, 2) + "\n"); -} - -// ------------------------------------------------------------------- fixtures -function makeFixture(tc) { - const dir = fs.mkdtempSync(path.join(TMP, "golden-")); - for (const [name, content] of Object.entries(tc.files ?? {})) { - const abs = path.join(dir, name); - fs.mkdirSync(path.dirname(abs), { recursive: true }); - fs.writeFileSync(abs, Buffer.from(String(content), "utf-8")); - } - for (const cmd of tc.fixture ?? []) { - const r = spawnSync("/bin/bash", ["-c", cmd], { cwd: dir, encoding: "utf-8" }); - if (r.status !== 0) { - throw new Error(`fixture command failed (${r.status}): ${cmd}\nstdout: ${r.stdout}\nstderr: ${r.stderr}`); - } - } - return dir; -} - -function stripCaseRuntimeFields(tc) { - const { files, fixture, ...rest } = tc; - return rest; -} - -// ================================================================== GROUP 1 -async function editGroup() { - const mod = await loadTs("packages/coding-agent/src/core/tools/edit.ts"); - - const gitless = { - files: { - "file.txt": "alpha\nbeta\ngamma\ndelta\n", - }, - }; - - const cases = [ - { - name: "unique-replacement", - ...gitless, - applyPrepareArguments: false, - input: { path: "file.txt", edits: [{ oldText: "beta", newText: "BETA" }] }, - }, - { - name: "two-edits", - files: { "file.txt": "alpha\nbeta\ngamma\ndelta\n" }, - applyPrepareArguments: false, - input: { - path: "file.txt", - edits: [ - { oldText: "alpha", newText: "ALPHA" }, - { oldText: "gamma", newText: "GAMMA" }, - ], - }, - }, - { - name: "not-found", - ...gitless, - applyPrepareArguments: false, - input: { path: "file.txt", edits: [{ oldText: "missing", newText: "x" }] }, - }, - { - name: "ambiguous", - files: { "file.txt": "repeat\nmiddle\nrepeat\n" }, - applyPrepareArguments: false, - input: { path: "file.txt", edits: [{ oldText: "repeat", newText: "x" }] }, - }, - { - name: "overlapping-edits", - files: { "file.txt": "abcdef\n" }, - applyPrepareArguments: false, - input: { - path: "file.txt", - edits: [ - { oldText: "abcd", newText: "X" }, - { oldText: "cdef", newText: "Y" }, - ], - }, - }, - { - name: "file-not-found", - applyPrepareArguments: false, - input: { path: "missing.txt", edits: [{ oldText: "a", newText: "b" }] }, - }, - { - name: "empty-edits", - ...gitless, - applyPrepareArguments: false, - input: { path: "file.txt", edits: [] }, - }, - { - name: "legacy-oldText-newText", - ...gitless, - applyPrepareArguments: true, - input: { path: "file.txt", oldText: "beta", newText: "BETA" }, - }, - { - name: "edits-as-json-string", - ...gitless, - applyPrepareArguments: true, - input: { path: "file.txt", edits: JSON.stringify([{ oldText: "beta", newText: "BETA" }]) }, - }, - { - name: "bom-file", - files: { "file.txt": "hello world\nsecond line\n" }, - applyPrepareArguments: false, - input: { path: "file.txt", edits: [{ oldText: "world", newText: "planet" }] }, - }, - { - name: "crlf-file", - files: { "file.txt": "line1\r\nline2\r\nline3\r\n" }, - applyPrepareArguments: false, - input: { path: "file.txt", edits: [{ oldText: "line2", newText: "LINE2" }] }, - }, - { - name: "fuzzy-trailing-whitespace", - files: { "file.txt": "alpha \nbeta\nkeep \n" }, - applyPrepareArguments: false, - input: { path: "file.txt", edits: [{ oldText: "alpha\n", newText: "ALPHA\n" }] }, - }, - { - name: "fuzzy-smart-quotes", - files: { "file.txt": "it’s a “test” line\n" }, - applyPrepareArguments: false, - input: { path: "file.txt", edits: [{ oldText: "it's a \"test\" line", newText: "plain quotes" }] }, - }, - { - name: "empty-oldtext", - ...gitless, - applyPrepareArguments: false, - input: { path: "file.txt", edits: [{ oldText: "", newText: "x" }] }, - }, - { - name: "no-change", - ...gitless, - applyPrepareArguments: false, - input: { path: "file.txt", edits: [{ oldText: "beta", newText: "beta" }] }, - }, - { - name: "directory-path", - fixture: ["mkdir -p adir"], - applyPrepareArguments: false, - input: { path: "adir", edits: [{ oldText: "a", newText: "b" }] }, - }, - ]; - - const out = []; - for (const tc of cases) { - const dir = makeFixture(tc); - const def = mod.createEditToolDefinition(dir); - let input = structuredClone(tc.input); - let preparedInput; - if (tc.applyPrepareArguments) { - input = def.prepareArguments(input); - preparedInput = input; - } - let result; - try { - const r = await def.execute("id", input, undefined); - result = { ok: true, text: r.content[0].text, details: r.details ?? null }; - } catch (err) { - result = { ok: false, error: err.message }; - } - let finalContent = null; - try { - finalContent = fs.readFileSync(path.resolve(dir, String(tc.input.path)), "utf-8"); - } catch { - finalContent = null; - } - out.push({ - name: tc.name, - files: tc.files ?? {}, - fixture: tc.fixture ?? [], - applyPrepareArguments: tc.applyPrepareArguments, - input: tc.input, - ...(preparedInput !== undefined ? { preparedInput } : {}), - result, - finalContent, - }); - } - writeCorpus("edit", out); - return out.length; -} - -// ================================================================== GROUP 2 -// Fixed commit dates keep commit hashes (printed by `git reset --hard`) -// deterministic across runs. -const FIXED_DATE = "GIT_AUTHOR_DATE='2005-04-07T22:13:13' GIT_COMMITTER_DATE='2005-04-07T22:13:13'"; - -const dirtyRepoFixture = [ - "git init -q .", - "git config user.email t@t", - "git config user.name t", - "echo original > tracked.txt", - "git add tracked.txt", - FIXED_DATE + " git commit -qm init", - "echo modified > tracked.txt", - "echo untracked > untracked.txt", -]; - -const cleanRepoFixture = [ - "git init -q .", - "git config user.email t@t", - "git config user.name t", - "echo original > tracked.txt", - "git add tracked.txt", - FIXED_DATE + " git commit -qm init", -]; - -function bashCase(name, opts) { - return { name, ...opts }; -} - -async function runBashCase(tc) { - let dir; - if (tc.cwdIsDeleted) { - dir = makeFixture({ files: tc.files ?? {}, fixture: tc.fixture ?? [] }); - fs.rmSync(dir, { recursive: true }); - } else { - dir = makeFixture(tc); - } - const bashMod = await loadTs("packages/coding-agent/src/core/tools/bash.ts"); - const def = bashMod.createBashToolDefinition(dir, tc.options); - const input = { command: tc.command }; - if (tc.timeout !== undefined) input.timeout = tc.timeout; - if (tc.allowDestructiveGit !== undefined) input.allowDestructiveGit = tc.allowDestructiveGit; - let result; - try { - const r = await def.execute("id", input, undefined); - result = { ok: true, isError: false, text: r.content[0].text, details: r.details ?? null }; - } catch (err) { - result = { ok: false, isError: true, error: err.message }; - } - const out = { - name: tc.name, - files: tc.files ?? {}, - fixture: tc.fixture ?? [], - command: tc.command, - ...(tc.timeout !== undefined ? { timeout: tc.timeout } : {}), - ...(tc.allowDestructiveGit !== undefined ? { allowDestructiveGit: tc.allowDestructiveGit } : {}), - ...(tc.envBypass ? { envBypass: tc.envBypass } : {}), - ...(tc.cwdIsDeleted ? { cwdIsDeleted: true } : {}), - result, - }; - // Normalize + probe details of fullOutputPath (existence is checked pre-normalization). - const rawPath = result.ok ? result.details?.fullOutputPath : undefined; - if (rawPath) { - let exists = false; - let bytes = null; - try { - const st = fs.statSync(rawPath); - exists = true; - bytes = st.size; - } catch { - exists = false; - } - result.fullOutputFile = { exists, bytes }; - } - return out; -} - -function bashCaseDefs() { - return [ - bashCase("echo-hello", { command: "echo hello" }), - bashCase("exit-code-3", { command: "echo out; exit 3" }), - bashCase("no-output", { command: "true" }), - bashCase("timeout", { command: "sleep 5", timeout: 1 }), - bashCase("stdout-stderr-both", { command: "echo out; echo err1 1>&2; echo out2; echo err2 1>&2" }), - bashCase("line-truncation", { command: "seq 1 2500" }), - bashCase("byte-truncation-multiline", { - command: "for i in $(seq 1 100); do head -c 600 /dev/zero | tr '\\0' x; echo; done", - }), - bashCase("single-huge-line", { - command: "python3 -c \"print('x'*60000)\"", - }), - bashCase("unicode-output", { - command: - "python3 -c \"import sys,time; sys.stdout.write('\\u00e9\\u00e9\\u00e9'); sys.stdout.flush(); time.sleep(0.05); sys.stdout.write('\\u2713'); sys.stdout.flush()\"", - }), - bashCase("destructive-checkout-dirty", { - fixture: dirtyRepoFixture, - command: "git checkout -- .", - }), - bashCase("destructive-allow-bypass", { - fixture: dirtyRepoFixture, - command: "git checkout -- .", - allowDestructiveGit: true, - }), - bashCase("destructive-reset-clean", { - fixture: cleanRepoFixture, - command: "git reset --hard", - }), - bashCase("destructive-quoted", { - command: "echo 'git reset --hard'", - }), - bashCase("destructive-cd-sub", { - fixture: [ - ...cleanRepoFixture, - "mkdir sub", - "cd sub && git init -q . && git config user.email t@t && git config user.name t && echo orig > subfile.txt && git add subfile.txt && " + FIXED_DATE + " git commit -qm subinit", - "cd sub && echo dirty > subfile.txt", - ], - command: "cd sub && git checkout -- .", - }), - bashCase("destructive-comment", { - fixture: cleanRepoFixture, - command: "echo done # git reset --hard", - }), - bashCase("env-bypass", { - fixture: dirtyRepoFixture, - command: "git checkout -- .", - envBypass: true, - }), - bashCase("cwd-missing", { - cwdIsDeleted: true, - command: "echo hi", - }), - ]; -} - -async function bashGroup() { - const defs = bashCaseDefs(); - const out = []; - // env-bypass needs PI_BASH_ALLOW_DESTRUCTIVE_GIT in process.env; run it in a - // child node process (getShellEnv() inherits process.env). - for (const tc of defs) { - if (!tc.envBypass) { - out.push(await runBashCase(tc)); - } - } - // Child run: only the env-bypass case. - const envBypassDef = defs.find((d) => d.envBypass); - const childEnv = { - ...process.env, - PI_BASH_ALLOW_DESTRUCTIVE_GIT: "1", - HARNESS_ONLY_BYPASS: "1", - HARNESS_OUT: path.join(CORPUS, "_env-bypass-part.json"), - }; - const r = spawnSync(process.execPath, [__filename], { encoding: "utf-8", env: childEnv, cwd: process.cwd() }); - if (r.status !== 0) { - throw new Error("env-bypass child run failed:\n" + r.stdout + "\n" + r.stderr); - } - const part = JSON.parse(fs.readFileSync(childEnv.HARNESS_OUT, "utf-8")); - fs.rmSync(childEnv.HARNESS_OUT, { force: true }); - out.push(part); - - writeCorpus("bash", out); - return out.length; -} - -// ================================================================== GROUP 3 -async function previewGroup() { - const cp = await loadTs("packages/coding-agent/src/core/tools/code-preview.ts"); - const inputs = [ - "npm run test -- --watch", - "git add -A && git commit -m x", - "python3 <<'EOF'\nprint('hello')\nEOF", - "%%bash\ncd sub && make", - "curl -s https://api.example.com/v1 -H 'Authorization: Bearer abc123def456'", - "cd sub && make", - "uv run python3 -m pytest tests/x.rs", - "tee -a out.log", - "API_TOKEN = \"abc123\"\nrequests.get(\"https://example.com\", headers={\"Authorization\": \"Bearer sk-live-0123456789\"})", - "python3 -c \"import base64; print(base64.b64encode(b'0123456789'*12))\"", - "cat data.bin", - "echo qwertyuiopasdfghjklzxcvbnm1234567890QWERTYUIOPASDFGHJKLZXCVBNM+/0987654321qazwsxedcrfvtgbyhnujmikolp", - "for i in $(ls /very/long/path/with/many/components/here); do echo processing $i; done", - "from pathlib import Path\nout = Path('out.txt')\nout.write_text('hello')\nimport subprocess\nsubprocess.run('git status', shell=True)", - "print('just printing')", - "%%bash\necho in-bash-cell", - ]; - const out = inputs.map((input) => ({ - input, - bash: cp.previewBashCommand(input), - ipython: cp.previewIpythonCode(input), - })); - writeCorpus("preview", out); - return out.length; -} - -// ================================================================== GROUP 4 -async function truncateGroup() { - const tr = await loadTs("packages/coding-agent/src/core/tools/truncate.ts"); - const cases = [ - { name: "exact-boundary-equal-limits", content: "a\nb\nc", maxLines: 3, maxBytes: 10 }, - { name: "one-line-over-head", content: "a\nb\nc\nd", maxLines: 3, maxBytes: 100 }, - { name: "one-line-over-tail", content: "a\nb\nc\nd", maxLines: 3, maxBytes: 100 }, - { - name: "multibyte-at-byte-boundary", - content: "éééééééééé\néééééééééé\néééééééééé\n", - maxLines: 100, - maxBytes: 24, - }, - { name: "first-line-exceeds-bytes-head", content: "xxxxxxxxxxxxxxxxxxxxxxxx\nshort\n", maxLines: 10, maxBytes: 16 }, - { name: "first-line-exceeds-bytes-tail", content: "xxxxxxxxxxxxxxxxxxxxxxxx\nshort\n", maxLines: 10, maxBytes: 16 }, - { - name: "tail-trailing-blank-partial-rescue", - content: "y".repeat(100) + "\n\n\n", - maxLines: 10, - maxBytes: 50, - }, - { name: "empty-string", content: "", maxLines: 5, maxBytes: 50 }, - { name: "no-trailing-newline", content: "a\nb\nc", maxLines: 2, maxBytes: 100 }, - { name: "byte-limit-tail-simple", content: Array.from({ length: 40 }, (_, i) => "line" + i).join("\n") + "\n", maxLines: 2000, maxBytes: 60 }, - ]; - const out = cases.map((tc) => ({ - name: tc.name, - content: tc.content, - maxLines: tc.maxLines, - maxBytes: tc.maxBytes, - head: tr.truncateHead(tc.content, { maxLines: tc.maxLines, maxBytes: tc.maxBytes }), - tail: tr.truncateTail(tc.content, { maxLines: tc.maxLines, maxBytes: tc.maxBytes }), - })); - writeCorpus("truncate", out); - return out.length; -} - -// ================================================================== GROUP 5 -async function ipythonGroup() { - const ip = await loadTs("packages/coding-agent/src/core/tools/ipython.ts"); - const kernelShared = await loadTs("packages/coding-agent/src/core/kernel/shared.ts"); - const BusyErr = kernelShared.KernelBusyAfterInterruptError; - - function makeProvisioner(executions) { - const calls = { ensure: 0, execute: 0, kill: 0 }; - let i = 0; - const manager = { - execute: async (code, opts) => { - calls.execute++; - const item = executions[Math.min(i++, executions.length - 1)]; - if (item instanceof Error) throw item; - return item; - }, - }; - return { - calls, - ensure: async (_onProgress, _signal) => { - calls.ensure++; - return manager; - }, - kill: async () => { - calls.kill++; - }, - }; - } - - const okResult = (over = {}) => ({ status: "ok", stdout: "", stderr: "", durationMs: 123, ...over }); - - const caseDefs = [ - { - name: "stdout-stderr-result-ordering", - code: "print('out')", - mockResults: [okResult({ stdout: "out\n", stderr: "err\n", result: "42" })], - ctxMode: "none", - }, - { - name: "error-status-traceback", - code: "1/0", - mockResults: [ - { - status: "error", - stdout: "before\n", - stderr: "", - result: undefined, - error: { - ename: "ZeroDivisionError", - evalue: "division by zero", - traceback: ["Traceback (most recent call last):", " File ...: division by zero"], - }, - durationMs: 5, - }, - ], - ctxMode: "none", - }, - { - name: "empty-output", - code: "x = 1", - mockResults: [okResult()], - ctxMode: "none", - }, - { - name: "background-output", - code: "import time; time.sleep(0.01)", - mockResults: [okResult({ stdout: "x", backgroundOutput: "late output\nfrom background" })], - ctxMode: "none", - }, - { - name: "aborted-status", - code: "import time; time.sleep(10)", - mockResults: [okResult({ status: "aborted", stdout: "partial output" })], - ctxMode: "none", - }, - { - name: "busy-then-cancel-no-ui", - code: "print('never')", - mockResults: [new BusyErr()], - ctxMode: "none", - }, - { - name: "busy-then-kill-restart-notice", - code: "print('after restart')", - mockResults: [new BusyErr(), okResult({ stdout: "after restart\n" })], - ctxMode: "ui-kill", - }, - ]; - - const out = []; - for (const tc of caseDefs) { - const prov = makeProvisioner(tc.mockResults); - const def = ip.createIpythonToolDefinition("/tmp", { provisioner: prov }); - const ctx = - tc.ctxMode === "ui-kill" - ? { - hasUI: true, - ui: { - async select(_prompt, _choices, _opts) { - return "Kill kernel and restart"; - }, - setWorkingMessage() {}, - }, - } - : undefined; - let result; - try { - const r = await def.execute("id", { code: tc.code }, undefined, undefined, ctx); - result = { ok: true, isError: r.isError === true, outputText: r.content[0].text }; - } catch (err) { - result = { ok: false, isError: true, error: err.message }; - } - out.push({ - name: tc.name, - code: tc.code, - ctxMode: tc.ctxMode, - mockResults: tc.mockResults.map((m) => (m instanceof Error ? { thrown: m.name, message: m.message } : m)), - provisionerCalls: prov.calls, - result, - }); - } - writeCorpus("ipython", out); - return out.length; -} - -// ================================================================== GROUP 6 -async function schemaGroup() { - const edit = await loadTs("packages/coding-agent/src/core/tools/edit.ts"); - const bash = await loadTs("packages/coding-agent/src/core/tools/bash.ts"); - const ipython = await loadTs("packages/coding-agent/src/core/tools/ipython.ts"); - const defs = { - bash: bash.createBashToolDefinition("/tmp"), - edit: edit.createEditToolDefinition("/tmp"), - ipython: ipython.createIpythonToolDefinition("/tmp"), - }; - const out = []; - for (const [tool, def] of Object.entries(defs)) { - out.push({ - tool, - name: def.name, - label: def.label, - description: def.description, - promptSnippet: def.promptSnippet, - ...(def.executionMode !== undefined ? { executionMode: def.executionMode } : {}), - parameters: def.parameters, - }); - } - writeCorpus("schema", out); - return out.length; -} - -// ======================================================================= main -async function main() { - fs.mkdirSync(CORPUS, { recursive: true }); - if (process.env.HARNESS_ONLY_BYPASS === "1") { - // Child run: execute only the env-bypass bash case, write it to HARNESS_OUT. - const defs = bashCaseDefs().filter((d) => d.envBypass); - const results = []; - for (const tc of defs) results.push(await runBashCase(tc)); - fs.writeFileSync(process.env.HARNESS_OUT, JSON.stringify(deepNorm(results[0]), null, 2) + "\n"); - process.exit(0); - } - const counts = {}; - counts.edit = await editGroup(); - counts.bash = await bashGroup(); - counts.preview = await previewGroup(); - counts.truncate = await truncateGroup(); - counts.ipython = await ipythonGroup(); - counts.schema = await schemaGroup(); - - const total = Object.values(counts).reduce((a, b) => a + b, 0); - console.log("corpus written:", JSON.stringify(counts), "total", total); -} - -main().catch((err) => { - console.error(err); - process.exit(1); -}); diff --git a/scripts/battery/README.md b/scripts/battery/README.md deleted file mode 100644 index e322b1a8655..00000000000 --- a/scripts/battery/README.md +++ /dev/null @@ -1,62 +0,0 @@ -# Perf-wave battery - -The perf wave measures the interactive TUI against the deployed TS -binary (the "ts" side) and a release build of this repo (the "rust" -side), gates the rust medians against the recorded baseline, and is -driven by the Makefile `perf-wave` target: - - make perf-wave - -`perf-wave` runs `ci_perf_wave.sh`, the CI-shaped entry: it creates an -ephemeral Prime sandbox (4 CPU / 16 GB, the spec class the baseline was -recorded on), installs the deployed TS binary plus a fresh release build -of the checkout, runs the wave there, downloads the results, and gates -them (`perf_gate.py` against `perf-baseline.json`). Every timing comes -from the same quiet machine for both sides, which is the whole point of -the sandbox. `PA_BENCH_NO_SANDBOX=1` runs the wave directly on the host -instead (`run_perf_wave.sh` + `perf_wave.py`). - -Files: - -- `ci_perf_wave.sh` — the Makefile entry (sandbox lifecycle, results - download, the gate call) -- `run_perf_wave.sh` — the local runner (same shape as the CI entry, - without the sandbox dance) -- `perf_wave.py` — the driver (dimensions below; accepts --ts-bin / - --rust-bin) -- `sustained_cpu.py` — the sustained-CPU dimension: the interactive TUI - driven through the faux provider — a 30s idle window, a paced plain-text - stream, a paced markdown/code stream, and a long tool-execution turn — - with %CPU sampled per process from /proc and ASSERTED: idle <5% and - streaming <30% of one core, keystroke-to-render mid-stream <50ms, the - paused viewport stable mid-stream, and a double Ctrl+C mid-scroll - terminating the process (bug #6's no-escape invariant) -- `stream_throughput.py` — the stream-throughput dimension (token - throughput of the live render loop, both binaries) -- `perf.py` — the perf measurement helpers the dimensions share: - startup launch-to-ready, keystroke-to-render typing latency, and - cold-resume-to-ready over the same tmux pane channel a real user - types through (both product sides) -- `scale_corpus.py` — the deterministic heavy-scale corpus the resume - dimension replays (the 350-turn / 1078-row corpus perf_wave.py - generates via scale_corpus for its resume dim) -- `perf_gate.py` — the gate (medians vs `perf-baseline.json`, default - threshold 1.10) -- `perf-baseline.json` — the recorded baseline the gate compares against -- `batterylib.py` — the shared harness (side sessions, the deterministic - mock provider, the daemon-reap sweep every harness reuses: - `reap_daemons(needles=[...])` sweeps any `--mode daemon` process whose - argv references the given paths — the graceful `sd` shutdown first, then - the SIGTERM/SIGKILL rounds until a respawned main is gone; the harness - sandboxes pin an isolated per-side TMPDIR so a supervisor's socket lands - under the sandbox root the sweep can name) -- `mock_provider.py` — the deterministic mock provider batterylib drives - (responses are session-scoped queues — a queue is selected per request - by markers in the request's user-message text, so a parent session and - a concurrently running spawned child each get their own scripted - responses deterministically) -- `ts_identity.py` — the shared ts-identity guard: every PATH-driven - harness refuses to run when its "ts" binary is this repo's Rust product -- `../ts_faux_extension.js` — the faux-provider driver extension the TS - side loads (scripted responses read from PRIME_AGENT_FAUX_SCRIPT); - verification harness only, never installed for real users diff --git a/scripts/battery/batterylib.py b/scripts/battery/batterylib.py deleted file mode 100644 index 55940f47650..00000000000 --- a/scripts/battery/batterylib.py +++ /dev/null @@ -1,715 +0,0 @@ -#!/usr/bin/env python3 -"""Shared helpers for the parity battery: isolated per-side environments, -tmux frame capture, a JSONL daemon-wire client, and evidence-file helpers. - -The battery drives the TS binary (ground truth) and the Rust binary side by -side against a deterministic mock provider (mock_provider.py). Nothing here -is part of the product. -""" - -from __future__ import annotations - -import json -import os -import re -import shutil -import socket -import subprocess -import time -from dataclasses import dataclass, field -from pathlib import Path - -NL = chr(10) -# Environment keys set by this box's own prime-agent session/worker; they must -# never leak into spawned daemons, workers, or interactive sessions. -SCRUB_ENV_PREFIXES = ("PRIME_AGENT_INTERNAL", "RLM_") -SCRUB_ENV_KEYS = ( - "TMUX", - "TMUX_PANE", - "PRIME_AGENT_SESSION_DIR", - "PRIME_AGENT_CODING_AGENT_DIR", - "PRIME_AGENT_CODING_AGENT_SESSION_DIR", - "PRIME_AGENT_KERNEL_OWNER_PID", - "PI_CODING_AGENT_DIR", - "PI_CODING_AGENT", -) - -TMUX_SIZE = (120, 36) - - -def scrubbed_env(agent_dir: Path, tmpdir: Path, extra: dict | None = None) -> dict: - """A clean env: no worker/role markers, isolated agent dir + TMPDIR.""" - env = { - k: v - for k, v in os.environ.items() - if not k.startswith(SCRUB_ENV_PREFIXES) and k not in SCRUB_ENV_KEYS - } - env["PRIME_AGENT_CODING_AGENT_DIR"] = str(agent_dir) - env["TMPDIR"] = str(tmpdir) - if extra: - env.update(extra) - return env - - -def rust_binary_staleness(binary: Path, repo_root: Path) -> str | None: - """Why a built Rust binary cannot include this checkout's current - product code (`None` = fresh or unknown). A stale build reports false - divergences against the TS ground truth (run 20260920: the - compaction-abort harness "failed" on main only because the checkout's - target/release binary predates the auto-compaction merge), so the - harnesses fail fast instead of wasting a run. The binary's mtime must - postdate the newest commit touching the product sources; set - PA_BATTERY_ALLOW_STALE_RUST=1 to skip the guard (deliberate - old-build testing). - """ - if os.environ.get("PA_BATTERY_ALLOW_STALE_RUST") == "1": - return None - try: - binary = Path(binary) - if not binary.exists(): - return None # a missing binary is the caller's own error - out = subprocess.run( - ["git", "log", "-1", "--format=%ct", "--", "crates", "prime-agent-runtime"], - cwd=repo_root, - capture_output=True, - text=True, - timeout=10, - ).stdout.strip() - if not out: - return None - newest_product_commit = int(out) - built = binary.stat().st_mtime - # One minute of slack: a build started at the commit's timestamp - # still includes it. - if built + 60 < newest_product_commit: - age_min = int((newest_product_commit - built) / 60) - return ( - f"{binary} predates the newest product commit in {repo_root} " - f"(binary built ~{age_min} min before the newest crates/ or " - "prime-agent-runtime/ change): a stale build reports false " - "divergences. Rebuild (cargo build --release -p pa-cli) or " - "point --rust-bin/PA_PARITY_RUST at a current build; set " - "PA_BATTERY_ALLOW_STALE_RUST=1 to override." - ) - except Exception: - return None - return None - - -def run_cmd( - argv: list[str], - env: dict, - cwd: Path, - timeout: float = 120.0, - stdin_text: str | None = "", -) -> dict: - """Run a command, returning a record with stdout/stderr/exit/duration.""" - start = time.time() - try: - proc = subprocess.run( - argv, - env=env, - cwd=str(cwd), - input=stdin_text, - capture_output=True, - text=True, - timeout=timeout, - ) - return { - "argv": argv, - "exit_code": proc.returncode, - "stdout": proc.stdout, - "stderr": proc.stderr, - "duration_s": round(time.time() - start, 2), - "timeout": False, - } - except subprocess.TimeoutExpired as exc: - return { - "argv": argv, - "exit_code": None, - "stdout": exc.stdout.decode() if isinstance(exc.stdout, bytes) else (exc.stdout or ""), - "stderr": exc.stderr.decode() if isinstance(exc.stderr, bytes) else (exc.stderr or ""), - "duration_s": round(time.time() - start, 2), - "timeout": True, - } - - -class MockProvider: - """One mock-provider process per side, so request logs never interleave.""" - - def __init__(self, battery_dir: Path, port_offset_holder: list[int]): - self.script_path = battery_dir / "mock-script.json" - self.requests_path = Path(str(self.script_path) + ".requests.jsonl") - for path in (self.script_path, self.requests_path): - if path.exists(): - path.unlink() - self.port = None - self._holder = port_offset_holder - self._proc = None - - def set_responses(self, responses: list[dict], queues: list[dict] | None = None) -> None: - """Write the mock script. `responses` is the default queue; each - entry of `queues` is a session-scoped queue ({"name", "match", - "matchModels", "responses"}) served to requests whose model id - (matchModels) or user-message text (match markers) selects it - (mock_provider.py picks per request). An entry may also be a - scripted provider failure: {"error": "", "status": 400}.""" - script = {"responses": responses} - if queues: - script["queues"] = queues - with open(self.script_path, "w") as f: - json.dump(script, f) - - def start(self) -> int: - here = Path(__file__).parent - for _ in range(20): - self._proc = subprocess.Popen( - ["python3", str(here / "mock_provider.py"), str(self.script_path)], - stdout=subprocess.PIPE, - stderr=subprocess.DEVNULL, - text=True, - ) - line = self._proc.stdout.readline() - try: - self.port = int(line.strip()) - return self.port - except ValueError: - self._proc.kill() - time.sleep(0.1) - raise RuntimeError("mock provider failed to start") - - def url(self) -> str: - return f"http://127.0.0.1:{self.port}/v1" - - def requests(self) -> list[dict]: - if not self.requests_path.exists(): - return [] - out = [] - for line in self.requests_path.read_text().splitlines(): - if line.strip(): - out.append(json.loads(line)) - return out - - def stop(self) -> None: - if self._proc: - self._proc.terminate() - try: - self._proc.wait(timeout=5) - except subprocess.TimeoutExpired: - self._proc.kill() - - -@dataclass -class Side: - """One product side (ts or rust) with fully isolated state.""" - - name: str - binary: str - root: Path # evidence/ - agent_dir: Path - work_dir: Path - daemon_socket: Path - mock: MockProvider - env: dict = field(default_factory=dict) - daemon_proc: subprocess.Popen | None = None - - @property - def base_url(self) -> str: - return self.mock.url() - - def write_models_json(self) -> None: - self.agent_dir.mkdir(parents=True, exist_ok=True) - models = { - "providers": { - "prime-inference": { - "api": "openai-completions", - "baseUrl": self.base_url, - "apiKey": "sk-battery", - "models": [ - { - "id": "mock-1", - "name": "Mock 1", - "api": "openai-completions", - "baseUrl": self.base_url, - "contextWindow": 128000, - "maxTokens": 4096, - } - ], - } - } - } - (self.agent_dir / "models.json").write_text(json.dumps(models, indent=1)) - - # -- evidence helpers --------------------------------------------------- - - def evidence(self, flow: str, name: str, text: str) -> Path: - path = self.root / flow / name - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(text) - return path - - def evidence_json(self, flow: str, name: str, obj) -> Path: - path = self.root / flow / name - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(obj, indent=1, sort_keys=False)) - return path - - def sessions_dir(self) -> Path: - return self.agent_dir / "sessions" - - def session_files(self) -> list[Path]: - d = self.sessions_dir() - if not d.exists(): - return [] - return sorted(d.glob("*.jsonl"), key=lambda p: p.stat().st_mtime) - - # -- daemon -------------------------------------------------------------- - - def start_daemon(self, timeout: float = 30.0) -> None: - """Start the daemon on this side's socket (identical argv both sides).""" - log = self.root / "daemon.log" - log.parent.mkdir(parents=True, exist_ok=True) - with open(log, "ab") as logfile: - self.daemon_proc = subprocess.Popen( - [self.binary, "--mode", "daemon", "--daemon-socket", str(self.daemon_socket)], - env=self.env, - cwd=str(self.work_dir), - stdin=subprocess.DEVNULL, - stdout=logfile, - stderr=logfile, - ) - deadline = time.time() + timeout - while time.time() < deadline: - if self.daemon_socket.exists(): - try: - with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as probe: - probe.connect(str(self.daemon_socket)) - return - except OSError: - pass - if self.daemon_proc.poll() is not None: - raise RuntimeError( - f"{self.name} daemon exited early: {log.read_text(errors='replace')[-2000:]}" - ) - time.sleep(0.2) - raise RuntimeError(f"{self.name} daemon socket never appeared") - - def stop_daemon(self) -> None: - """Shut this side's daemon down and reap its whole tree (the - shared `reap_daemons` sweep; see its docstring for the leak - history). Only processes whose argv references this side's unique - socket paths are touched, never unrelated daemons.""" - needles = [str(self.daemon_socket)] - tmpdir = self.env.get("TMPDIR") - if tmpdir: - needles.append(tmpdir.rstrip("/")) - reap_daemons(socket_paths=[self.daemon_socket], needles=needles, proc=self.daemon_proc) - - def own_daemon_pids(self) -> list[int]: - """Daemon processes whose argv references this side's daemon - socket or its TMPDIR (the supervisor's socket lives there).""" - exclude = {self.daemon_proc.pid} if self.daemon_proc else set() - needles = [str(self.daemon_socket)] - tmpdir = self.env.get("TMPDIR") - if tmpdir: - needles.append(tmpdir.rstrip("/")) - return daemon_pids_matching(needles, exclude_pids=exclude) - - -def daemon_pids_matching(needles, exclude_pids=()) -> list[int]: - """Live `--mode daemon` processes whose argv references any of - `needles` (daemon socket paths, isolated TMPDIR roots, or a harness - sandbox root containing either). Empty-string needles are dropped: a - bare "" would match every process. The scan walks /proc on Linux and - `ps` on macOS (a harness must reap on both).""" - seen: list[int] = [] - live_needles = [needle for needle in needles if needle] - if Path("/proc").is_dir(): - for proc_dir in Path("/proc").iterdir(): - if not proc_dir.name.isdigit(): - continue - pid = int(proc_dir.name) - if pid in (1, os.getpid()) or pid in exclude_pids: - continue - try: - argv = [ - part.decode(errors="replace") - for part in (proc_dir / "cmdline").read_bytes().split(b"\0") - if part - ] - except OSError: - continue - if "--mode" not in argv or "daemon" not in argv: - continue - if any(needle in part for part in argv for needle in live_needles): - seen.append(pid) - return seen - scan = subprocess.run( - ["ps", "-axo", "pid=,command="], - capture_output=True, - text=True, - env=dict(os.environ, COLUMNS="4096"), - ) - for line in scan.stdout.splitlines(): - parts = line.strip().split(None, 1) - if len(parts) < 2 or not parts[0].isdigit(): - continue - pid = int(parts[0]) - if pid in (1, os.getpid()) or pid in exclude_pids: - continue - argv = parts[1].split() - if "--mode" not in argv or "daemon" not in argv: - continue - # Needles can span argv splits (a socket, sandbox, or TMPDIR path - # with spaces), so they match against the full rendered command - # line, never against the fragments. - if any(needle in parts[1] for needle in live_needles): - seen.append(pid) - return seen - - -def worker_pids_under(cwd_roots=(), exclude_pids=()) -> list[int]: - """Processes whose working directory sits under any of `cwd_roots` - (normalized, trailing-separator-safe). The products' detached session - workers run as ` worker` with no sandbox path in their argv, - but with the session's cwd (the side's work dir), so cwd is the only - scope that reaches them. /proc-only: the macOS fallback reaps the - daemon pair (the argv scan above) and leaves the detached workers to - their own exit paths.""" - if not Path("/proc").is_dir(): - return [] - roots = [] - for root in cwd_roots: - if root: - text = str(root).rstrip("/") + "/" - roots.append(text) - seen: list[int] = [] - for proc_dir in Path("/proc").iterdir(): - if not proc_dir.name.isdigit(): - continue - pid = int(proc_dir.name) - if pid in (1, os.getpid()) or pid in exclude_pids: - continue - try: - cwd = os.readlink(proc_dir / "cwd") - except OSError: - continue - if any(cwd.startswith(root) for root in roots): - seen.append(pid) - return seen - - -def reap_daemons(socket_paths=(), needles=(), proc=None, cwd_roots=()) -> None: - """Reap a harness's daemon tree (the #221 pattern, lifted so every - standalone harness shares it). - - A bare terminate() leaks: the TS product runs a per-run supervisor - (its own `--mode daemon` process on a socket under the side's TMPDIR) - that RESPAWNS a killed main daemon, so SIGTERM alone leaves the pair - behind (run 20260920: every wire-harness pass leaked two daemons; the - #223 close-out: a standalone harness rmtree'd its tempdir and left a - daemon spinning at 74% CPU on the deleted socket). The graceful `sd` wire - shutdown (both products implement it) goes first for every - `socket_paths` entry; `proc` (a directly spawned daemon) is then - terminated; then the sweep kills every `--mode daemon` process whose - argv references any of `needles` — a respawned main or a detached - supervisor — and finally every process whose cwd sits under any of - `cwd_roots` (the detached session workers: ` worker` carries - no sandbox path in its argv, only in its cwd). SIGTERM first, - escalate to SIGKILL, re-check for a respawned main until the - supervisor is gone.""" - for socket_path in socket_paths: - try: - wire = Wire(Path(socket_path)) - wire.send_command("sd", {"type": "shutdown"}) - wire.close() - time.sleep(2) - except Exception: - pass - if proc and proc.poll() is None: - proc.terminate() - try: - proc.wait(timeout=10) - except subprocess.TimeoutExpired: - proc.kill() - exclude = {proc.pid} if proc else set() - for _ in range(3): - pids = daemon_pids_matching(needles, exclude_pids=exclude) - pids += worker_pids_under(cwd_roots, exclude_pids=exclude) - if not pids: - return - for pid in pids: - try: - os.kill(pid, 15) - except (ProcessLookupError, PermissionError): - pass - time.sleep(1.0) - for pid in daemon_pids_matching(needles, exclude_pids=exclude) + worker_pids_under( - cwd_roots, exclude_pids=exclude - ): - try: - os.kill(pid, 9) - except (ProcessLookupError, PermissionError): - pass - time.sleep(0.5) - - -class Wire: - """JSONL daemon-protocol client (protocol 7, the TS wire format).""" - - def __init__(self, socket_path: Path): - self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - self.sock.settimeout(90) - self.sock.connect(str(socket_path)) - self.buf = b"" - self.events: list = [] - self.hello = self.read_line() - - def send_command(self, command_id: str, command: dict) -> None: - envelope = { - "type": "command", - "id": command_id, - "protocol": {"name": "prime-agent.daemon", "version": 7}, - "command": command, - } - self.sock.sendall((json.dumps(envelope) + chr(10)).encode()) - - def read_line(self, timeout: float = 60.0) -> dict: - """Read one JSON line; raises TimeoutError when nothing arrives.""" - deadline = time.time() + timeout - while bytes([10]) not in self.buf: - remaining = deadline - time.time() - if remaining <= 0: - raise TimeoutError("daemon line timeout") - self.sock.settimeout(max(0.1, remaining)) - chunk = self.sock.recv(65536) - if not chunk: - raise EOFError("daemon closed the connection") - self.buf += chunk - line, self.buf = self.buf.split(bytes([10]), 1) - return json.loads(line.decode()) - - def request(self, command_id: str, command: dict, timeout: float = 60.0) -> dict: - """Send a command; return the response carrying the matching id. - - Side events (non-matching lines) are recorded into `self.events`. - """ - self.send_command(command_id, command) - deadline = time.time() + timeout - while True: - remaining = deadline - time.time() - if remaining <= 0: - raise TimeoutError(f"no response for command {command_id}") - line = self.read_line(timeout=remaining) - if line.get("id") == command_id: - return line - self.events.append(line) - - def drain(self, timeout: float = 3.0) -> list: - """Read whatever the daemon sends for up to `timeout` seconds, - buffering side events into `self.events`; returns the new events.""" - deadline = time.time() + timeout - fresh = [] - while True: - remaining = deadline - time.time() - if remaining <= 0: - return fresh - try: - line = self.read_line(timeout=remaining) - except TimeoutError: - return fresh - fresh.append(line) - self.events.append(line) - - def close(self) -> None: - try: - self.sock.close() - except OSError: - pass - - -# -- tmux --------------------------------------------------------------------- - -def tmux(*args: str, check: bool = True) -> subprocess.CompletedProcess: - """Run tmux on the default socket (never the agentui socket).""" - env = {k: v for k, v in os.environ.items() if k not in ("TMUX", "TMUX_PANE")} - proc = subprocess.run( - ["env", "-u", "TMUX", "-u", "TMUX_PANE", "tmux", *args], - env=env, - capture_output=True, - text=True, - ) - if check and proc.returncode != 0: - raise RuntimeError(f"tmux {args} failed: {proc.stderr}") - return proc - - -def tmux_launch( - session: str, - command: list[str], - env: dict, - cwd: Path, - size: tuple[int, int] = TMUX_SIZE, - unset_keys: tuple[str, ...] = (), -) -> None: - """Create a detached session of `size` running `command` with `env`. - - tmux panes inherit the tmux server's environment, not the client's, so - the pane command is wrapped in `env KEY=VALUE ...` (and TMUX unset) to - guarantee isolation from the ambient agent session. - - `unset_keys` names keys the pane must never see even though they - inherit from the server (dropping them from `env` is not enough: an - inherited key is neither an assignment nor on the scrub list). - - The pane runs with `-c cwd` as its working directory, so a relative - `command[0]` (e.g. `target/release/prime-agent` passed from the repo - root) would resolve against the pane cwd and vanish instantly. Resolve - the binary to an absolute path before it reaches the pane. - """ - if command and not os.path.isabs(command[0]) and "/" in command[0]: - resolved = Path(command[0]).resolve() - if not resolved.exists(): - raise RuntimeError(f"tmux_launch: binary {command[0]} not found at {resolved}") - command = [str(resolved), *command[1:]] - # The pane inherits the tmux server env, so only the deltas (the scrubbed - # overrides) need explicit assignment; everything else stays inherited. - assignment = [ - f"{key}={value}" - for key, value in sorted(env.items()) - if os.environ.get(key) != value or key in ("PRIME_AGENT_CODING_AGENT_DIR", "TMPDIR") - ] - unset = ["-u", "TMUX", "-u", "TMUX_PANE"] - for key in SCRUB_ENV_KEYS: - if key not in ("TMUX", "TMUX_PANE"): - unset += ["-u", key] - # The tmux server inherited this session's worker markers; unset every - # one of them so pane processes never see daemon-worker identity. - for key in list(os.environ): - if key.startswith(SCRUB_ENV_PREFIXES): - unset += ["-u", key] - # The caller's keys ride the same unset (a key that only inherits from - # the server reaches the pane unless it is named here). - for key in unset_keys: - unset += ["-u", key] - # tmux runs multi-argument pane commands through its default shell, so - # wrap explicitly: sh -c 'exec env -u ... KEY=V ... '. - words = ( - ["exec", "/usr/bin/env"] - + unset - + assignment - + list(command) - ) - import shlex - - shell_command = " ".join(shlex.quote(w) for w in words) - tmux( - "new-session", - "-d", - "-x", - str(size[0]), - "-y", - str(size[1]), - "-s", - session, - "-c", - str(cwd), - "/bin/sh", - "-c", - shell_command, - ) - - -def tmux_resize(session: str, size: tuple[int, int]) -> None: - """Resize a detached session's window (frame-parity captures at a - second terminal size).""" - tmux("resize-window", "-t", session, "-x", str(size[0]), "-y", str(size[1]), check=False) - - -def tmux_capture(session: str, pane: str = "0") -> str: - return tmux("capture-pane", "-p", "-t", f"{session}:{pane}").stdout - - -def tmux_send(session: str, keys: str, enter: bool = True) -> None: - tmux("send-keys", "-t", session, keys, "Enter" if enter else "") - - -def children_of(pid: int) -> list[int]: - """Live child pids of `pid` (youngest last), by /proc traversal.""" - out = [] - for proc_dir in Path("/proc").iterdir(): - if not proc_dir.name.isdigit(): - continue - try: - stat = (proc_dir / "stat").read_text() - except OSError: - continue - # The comm field can contain spaces/parens: split after the last ')'. - rest = stat[stat.rfind(")") + 1 :].split() - if len(rest) >= 2 and rest[1] == str(pid): - out.append(int(proc_dir.name)) - return sorted(out) - - -def tmux_kill(session: str) -> None: - tmux("kill-session", "-t", session, check=False) - - -def tmux_wait_text(session: str, pattern: str, timeout: float = 60.0, poll: float = 0.5) -> str: - """Poll a pane until `pattern` appears; returns the final frame.""" - deadline = time.time() + timeout - frame = "" - while time.time() < deadline: - frame = tmux_capture(session) - if re.search(pattern, frame): - return frame - time.sleep(poll) - return frame - - -# -- normalization + diff helpers --------------------------------------------- - -def normalize(value, skip_keys=("timestamp", "id", "parentId", "sessionId", "socketPath", "pid")): - """Recursively normalize volatile keys for cross-side comparisons.""" - - def norm(v): - if isinstance(v, dict): - return {k: ("" if k in skip_keys else norm(val)) for k, val in v.items()} - if isinstance(v, list): - return [norm(x) for x in v] - return v - - return norm(value) - - -def shape_of(value): - """A structural fingerprint: dict keys / list shapes / type names.""" - - def shp(v): - if isinstance(v, dict): - return {k: shp(val) for k, val in sorted(v.items())} - if isinstance(v, list): - return [shp(v[0])] if v else [] - return type(v).__name__ - - return shp(value) - - -def copy_dir_contents(src: Path, dst: Path) -> None: - if not src.exists(): - return - if dst.exists(): - shutil.rmtree(dst) - shutil.copytree(src, dst) - - -def write_report(path: Path, title: str, sections: list[dict]) -> None: - """Write the markdown gap report.""" - lines = [f"# {title}", ""] - for section in sections: - lines.append(f"## {section['title']}") - lines.append("") - for row in section.get("rows", []): - lines.append(f"- {row}") - lines.append("") - path.write_text(NL.join(lines)) diff --git a/scripts/battery/ci_perf_wave.sh b/scripts/battery/ci_perf_wave.sh deleted file mode 100755 index 1b29a9cab99..00000000000 --- a/scripts/battery/ci_perf_wave.sh +++ /dev/null @@ -1,106 +0,0 @@ -#!/usr/bin/env bash -# CI entry for the perf wave (driven by the benchmark workflow on main's tree). -# -# Creates an ephemeral Prime sandbox (4 CPU / 16 GB, the same spec class the -# baseline was recorded on), installs the deployed TS binary plus a fresh -# release build of the checkout, runs scripts/battery/run_perf_wave.sh -# there, downloads the results, gates them against the recorded baseline, -# and terminates the sandbox. Every timing comes from the same quiet -# machine for both sides, which is the whole point of the sandbox. -# -# Requirements on the runner: the `prime` CLI, docker-image access to -# rust:1, tmux/python3 inside the sandbox image, network access. -# -# Environment: -# PA_BENCH_NO_SANDBOX=1 run the wave directly on this runner instead -# PRIME_SANDBOX_NAME sandbox name (default build-perf-benchmark) -# PRIME_BENCH_TRIALS trial count (default 5) -set -euo pipefail -cd "$(dirname "$0")/../.." - -SANDBOX_NAME="${PRIME_SANDBOX_NAME:-build-perf-benchmark}" -TRIALS="${PRIME_BENCH_TRIALS:-5}" - -if [ -n "${PA_BENCH_NO_SANDBOX:-}" ]; then - PRIME_BENCH_TRIALS="$TRIALS" scripts/battery/run_perf_wave.sh - exit $? -fi - -SANDBOX_ID="$( - PRIME_DISABLE_VERSION_CHECK=1 prime sandbox --plain create \ - --name "$SANDBOX_NAME" rust:1 \ - --cpu-cores 4 --memory-gb 16 --disk-size-gb 80 \ - --timeout-minutes 300 -y | - grep -oE '[0-9a-z]{20,}' | tail -1 -)" -cleanup() { - PRIME_DISABLE_VERSION_CHECK=1 prime sandbox --plain delete "$SANDBOX_ID" >/dev/null 2>&1 || true -} -trap cleanup EXIT -echo "== sandbox $SANDBOX_ID" - -run_in_sandbox() { - # 300s exec cap: long work runs detached with a DONE-marker poll. - PRIME_DISABLE_VERSION_CHECK=1 prime sandbox --plain run "$SANDBOX_ID" "$1" -} - -# 1. ship the checkout + the deployed TS release into the sandbox. -git bundle create /tmp/benchmark.bundle --all -prime sandbox --plain upload "$SANDBOX_ID" /tmp/benchmark.bundle /tmp/benchmark.bundle -RELEASE_TREE="$(dirname "$(readlink -f "$(command -v prime-agent)")")/.." -# shellcheck disable=SC2012 -TS_RELEASE_TAR=/tmp/ts-release.tar -tar cf "$TS_RELEASE_TAR" -C "$RELEASE_TREE" . -prime sandbox --plain upload "$SANDBOX_ID" "$TS_RELEASE_TAR" /tmp/ts-release.tar - -cat > /tmp/sb-benchmark.sh <<'EOF' -#!/bin/bash -set -euo pipefail -mkdir -p /opt/repo /opt/ts-release -cd /opt/repo -git clone /tmp/benchmark.bundle repo -cd repo -git checkout "${GITHUB_HEAD_REF:-main}" -git log --oneline -1 -tar xf /tmp/ts-release.tar -C /opt/ts-release -TS_BIN=$(find /opt/ts-release -maxdepth 3 -name prime-agent -type f | head -1) -ln -sf "$TS_BIN" /usr/local/bin/prime-agent -prime-agent --version | head -1 -apt-get update -qq && apt-get install -y -qq tmux procps time -# 2. fresh release build + the wave, detached (the 300s exec cap). -nohup bash -c ' - cd /opt/repo/repo && - PRIME_BENCH_TRIALS="'"$PRIME_BENCH_TRIALS"'" \ - PA_WAVE_RUST_BIN=/opt/repo/repo/target/release/prime-agent \ - scripts/battery/run_perf_wave.sh - echo $? > /opt/wave.exit -' > /opt/wave.log 2>&1 & -echo detached-started -EOF -prime sandbox --plain upload "$SANDBOX_ID" /tmp/sb-benchmark.sh /tmp/sb-benchmark.sh - -# 3. poll the detached wave to completion. -DEADLINE=$(( $(date +%s) + 14400 )) -while true; do - STATUS=$(run_in_sandbox "if [ -f /opt/wave.exit ]; then cat /opt/wave.exit; else echo running; fi" | tail -1 | tr -d '[:space:]') - [ "$STATUS" != "running" ] && break - if [ "$(date +%s)" -gt "$DEADLINE" ]; then - echo "FATAL: sandbox wave exceeded the 4h poll deadline" >&2 - exit 2 - fi - sleep 60 -done -run_in_sandbox "tail -n 40 /opt/wave.log" || true -if [ "$STATUS" != "0" ]; then - echo "FATAL: sandbox wave exited with $STATUS (log above)" >&2 - exit 2 -fi - -# 4. pull the results back and gate them on the runner. -mkdir -p scripts/battery/runs -RUN_DIR="scripts/battery/runs/perf-wave-$(date -u +%Y%m%dT%H%M%SZ)" -mkdir -p "$RUN_DIR" -RUN_IN_SANDBOX_DIR=$(run_in_sandbox "ls -1dt /opt/repo/repo/scripts/battery/runs/perf-wave-* | head -1" | grep perf-wave | head -1) -prime sandbox --plain download "$SANDBOX_ID" "$RUN_IN_SANDBOX_DIR/results.json" "$RUN_DIR/results.json" -python3 scripts/battery/perf_gate.py --results "$RUN_DIR/results.json" -echo "== wave results: $RUN_DIR" diff --git a/scripts/battery/mock_provider.py b/scripts/battery/mock_provider.py deleted file mode 100644 index e392238012a..00000000000 --- a/scripts/battery/mock_provider.py +++ /dev/null @@ -1,297 +0,0 @@ -#!/usr/bin/env python3 -"""Deterministic OpenAI-compatible mock provider for the parity battery. - -Serves `/v1/chat/completions` (SSE streaming) and `/v1/models` from a JSON -script file, so the TS binary and the Rust binary can be driven side by side -with identical model responses. Not part of the product; battery harness -only. - -Script file format: - { - "responses": [ - {"text": "hello"}, - {"toolCall": {"name": "bash", "arguments": {"command": "echo hi"}}}, - {"error": "prompt is too long: 213462 tokens > 200000 maximum", "status": 400}, - {"text": "done", "delayMs": 500, "usage": {"prompt_tokens": 100, "completion_tokens": 10, "total_tokens": 110, "prompt_tokens_details": {"cached_tokens": 80}}} - ], - "queues": [ - {"name": "child", "match": ["child task text"], "matchModels": ["mock-1"], "responses": [{"text": "child reply"}]} - ] - } -Optional per-response keys: `delayMs` (stream starts after the delay), `usage` -(overrides the reported token usage, so a flow can push the session past a -context/compaction threshold deterministically), and `error`/`status` (a -scripted provider failure: the non-2xx answer carries the OpenAI error body, -so a flow can script overflow probes identically for both binaries). - -Responses are SESSION-SCOPED, not global. The chat-completions wire carries -no session id, so a queue is selected per request: the first queue whose -`matchModels` entry equals the request's model id, or whose `match` marker -appears in the concatenated text of the request's user-role messages, serves -the request; a request that matches no queue falls through to the default -`responses` queue. This keeps a parent session and a spawned child session -(which race the provider concurrently) on independent scripted response -cursors instead of popping one shared queue in arrival order, and keeps -model-routed flows (a scripted turn model vs a dashboard status-line model) -off each other's cursors. User-role text is the discriminator because a -parent's post-tool continuation embeds the child's task text in tool-call -arguments and tool results; matching those would misroute the parent. -Each queue pops its next scripted response per matching request -(round-robin when a queue runs dry: its last entry repeats). Every request, -the raw request body, and the serving queue's name are logged to -`