diff --git a/.github/workflows/solheim-provider-smoke.yml b/.github/workflows/solheim-provider-smoke.yml index 7e0298818c..e79036bcea 100644 --- a/.github/workflows/solheim-provider-smoke.yml +++ b/.github/workflows/solheim-provider-smoke.yml @@ -41,6 +41,7 @@ jobs: pnpm --filter @roo-code/vscode-webview build - name: Install display dependencies + timeout-minutes: 5 run: sudo apt-get update && sudo apt-get install -y xvfb ffmpeg libasound2t64 - name: Run Solheim provider smoke @@ -73,6 +74,7 @@ jobs: path: | ${{ runner.temp }}/solheim-provider-smoke/verdict.json ${{ runner.temp }}/solheim-provider-smoke/trace.json + ${{ runner.temp }}/solheim-provider-smoke/sidebar-focus.json if-no-files-found: error retention-days: 7 diff --git a/scripts/agent-evals/README.md b/scripts/agent-evals/README.md index bd24b6cfba..6eef1d6b0b 100644 --- a/scripts/agent-evals/README.md +++ b/scripts/agent-evals/README.md @@ -30,6 +30,11 @@ Execution limits are enforced, not decorative. The controller applies the activation, task start, task run, usage drain, and teardown step limits, and the trace records the measured durations. +Two launch options are off by default. `testsPath` adds `--extensionTestsPath` +to the VS Code launch; that module's `run()` must never settle, because VS Code +exits once it does. `holdBeforeTeardownMs` holds the finished task on screen +for that window after the completion message, before teardown starts. + ## Structured trace Each run writes `trace.json` with `traceSchemaVersion: 1`. The trace is diff --git a/scripts/agent-evals/isolation.ts b/scripts/agent-evals/isolation.ts index 06ab950ef6..17af4a9ae3 100644 --- a/scripts/agent-evals/isolation.ts +++ b/scripts/agent-evals/isolation.ts @@ -19,12 +19,21 @@ export function buildChildEnvironment( } } -export function buildLaunchArgs(workspace: string, userData: string, extensions: string, extension: string): string[] { +export function buildLaunchArgs( + workspace: string, + userData: string, + extensions: string, + extension: string, + testsPath?: string, +): string[] { return [ workspace, `--user-data-dir=${userData}`, `--extensions-dir=${extensions}`, `--extensionDevelopmentPath=${extension}`, + // Off by default: an --extensionTestsPath module settles into a VS Code + // exit, so only a caller that owns a never-settling module may set it. + ...(testsPath ? [`--extensionTestsPath=${testsPath}`] : []), "--no-sandbox", "--disable-gpu-sandbox", "--password-store=basic", diff --git a/scripts/agent-evals/launcher.ts b/scripts/agent-evals/launcher.ts index 1edd2917d2..bfcd92ed0e 100644 --- a/scripts/agent-evals/launcher.ts +++ b/scripts/agent-evals/launcher.ts @@ -17,6 +17,11 @@ export interface HostRuntimeOptions { artifactsDir: string now?: () => number pause?: (ms: number) => Promise + // Optional --extensionTestsPath module. The module's run() must never + // settle, because VS Code exits when it does. Off by default. + testsPath?: string + // Optional bounded status artifact written by the extension test module. + testsDiagnosticsPath?: string } const readCommit = async (root: string): Promise => { @@ -84,13 +89,19 @@ export function createHostRuntime(options: HostRuntimeOptions): AgentRuntime { path.join(temp, "userdata"), path.join(temp, "extensions"), extension, + options.testsPath, ) const child = spawn( process.env.DISPLAY ? executable : "xvfb-run", process.env.DISPLAY ? args : ["-a", executable, ...args], { detached: true, - env: buildChildEnvironment(process.env, isolatedHome, socket), + env: { + ...buildChildEnvironment(process.env, isolatedHome, socket), + ...(options.testsDiagnosticsPath + ? { AGENT_EVALS_TESTS_DIAGNOSTICS_PATH: options.testsDiagnosticsPath } + : {}), + }, stdio: "ignore", }, ) diff --git a/scripts/agent-evals/runner.test.ts b/scripts/agent-evals/runner.test.ts index d65e56a144..376d07b540 100644 --- a/scripts/agent-evals/runner.test.ts +++ b/scripts/agent-evals/runner.test.ts @@ -252,6 +252,7 @@ function harness( signals, artifacts, emit, + close, interrupt: () => interrupt(), get cleaned() { return cleaned @@ -1050,4 +1051,94 @@ describe("agent fixture controller", () => { assert.equal(result.execution.sessionLost, false) assert.equal(result.teardown.status, "ok") }) + + for (const hostExited of [false, true]) { + it(`holds teardown only while the completed host is live (exited=${hostExited})`, async () => { + const fixtureName = `controller-hold-${hostExited}` + registerTestFixture(fixtureName) + const f = harness("no-events") + f.emit({ + eventName: RooCodeEventName.Message, + payload: [ + { + taskId: "root", + action: "created", + message: { ts: 1, type: "say", say: "completion_result", text: "synthetic final answer" }, + }, + ], + }) + f.emit({ + eventName: RooCodeEventName.TaskCompleted, + payload: [ + "root", + { totalTokensIn: 7, totalTokensOut: 3, totalCost: 0, contextTokens: 9 }, + {}, + { + isSubtask: false, + }, + ], + }) + const holdPauses: number[] = [] + let closeTaskSeenDuringHold = false + const pause = f.runtime.pause + f.runtime.pause = async (ms) => { + if (ms === 3_000) { + holdPauses.push(ms) + closeTaskSeenDuringHold = f.commands.some( + (command) => command.commandName === TaskCommandName.CloseTask, + ) + } + await pause(ms) + } + const result = await runAgentFixture(f.runtime, { + fixtureName, + credential: CREDENTIAL, + observers: { + onAcceptedEvent: (event) => { + if (hostExited && event.eventName === RooCodeEventName.TaskCompleted) f.close() + }, + shouldStop: () => true, + }, + holdBeforeTeardownMs: 3_000, + }) + assert.equal(result.execution.code, "COMPLETED") + assert.equal(result.execution.completionMessageSeen, true) + assert.deepEqual(holdPauses, hostExited ? [] : [3_000]) + // The hold runs before teardown, so CloseTask is sent only after it. + assert.equal(closeTaskSeenDuringHold, false) + assert.equal(f.commands.at(-1)?.commandName, TaskCommandName.CloseTask) + assert.equal(result.teardown.status, "ok") + assert.equal(result.teardown.hostExited, true) + const trace = writtenTrace(f) + // CloseTask exits immediately, so none of the recording hold belongs to teardown. + assert.equal(trace.execution.timings.teardownMs, 0) + }) + } + it("skips the teardown hold without a completion message", async () => { + registerTestFixture("controller-hold-skipped", { + limits: { + activationTimeoutMs: 120_000, + taskStartTimeoutMs: 35_000, + taskRunTimeoutMs: 200, + usageDrainMs: 100, + teardownStepMs: 100, + }, + }) + const f = harness("no-events") + const holdPauses: number[] = [] + const pause = f.runtime.pause + f.runtime.pause = async (ms) => { + if (ms === 3_000) holdPauses.push(ms) + await pause(ms) + } + const result = await runAgentFixture(f.runtime, { + fixtureName: "controller-hold-skipped", + credential: CREDENTIAL, + holdBeforeTeardownMs: 3_000, + }) + assert.equal(result.execution.code, "TASK_RUN_TIMEOUT") + assert.equal(result.execution.completionMessageSeen, false) + assert.deepEqual(holdPauses, []) + assert.equal(result.teardown.status, "ok") + }) }) diff --git a/scripts/agent-evals/runner.ts b/scripts/agent-evals/runner.ts index 5c12961db3..c671dabc83 100644 --- a/scripts/agent-evals/runner.ts +++ b/scripts/agent-evals/runner.ts @@ -57,6 +57,10 @@ export interface AgentRunOptions { fixtureName: string credential: string observers?: AgentRunObservers + // Optional hold before teardown after the completion message, so a + // recording can capture the finished task. Off by default; skipped when + // no completion message was seen or the host is already gone. + holdBeforeTeardownMs?: number } export interface AgentRunResult { @@ -325,6 +329,11 @@ export async function runAgentFixture(runtime: AgentRuntime, options: AgentRunOp // Never expose thrown provider, filesystem or host messages. } finally { waiter?.onExit() + // A recording needs the finished task on screen before teardown + // clears it. The hold is opt-in and only applies after the + // completion message, while a live host can still show it. + if (completionMessageSeen && options.holdBeforeTeardownMs && host && !host.exited) + await runtime.pause(options.holdBeforeTeardownMs) const teardownStart = runtime.now() const attempt = async (step: string, operation: () => unknown) => { try { diff --git a/scripts/solheim-smoke-workflow.test.mjs b/scripts/solheim-smoke-workflow.test.mjs index b6656d6aef..b268ec4a3e 100644 --- a/scripts/solheim-smoke-workflow.test.mjs +++ b/scripts/solheim-smoke-workflow.test.mjs @@ -34,6 +34,12 @@ describe("Solheim provider smoke workflow policy", () => { assert.equal(checkouts[0].with["persist-credentials"], false) }) + it("bounds the display dependency install to five minutes", () => { + const apt = steps.find((step) => step.name === "Install display dependencies") + assert.ok(apt, "missing display dependency step") + assert.equal(apt["timeout-minutes"], 5) + }) + it("exposes the provider credential only to the bounded provider step", () => { const provider = steps.find((step) => step.id === "provider") assert.ok(provider, "missing provider step") @@ -61,7 +67,10 @@ describe("Solheim provider smoke workflow policy", () => { .split("\n") .map((line) => line.trim()), ) - assert.deepEqual(paths, [[output + "/verdict.json", output + "/trace.json"], [output + "/smoke.mp4"]]) + assert.deepEqual(paths, [ + [output + "/verdict.json", output + "/trace.json", output + "/sidebar-focus.json"], + [output + "/smoke.mp4"], + ]) for (const step of uploads) { assert.equal( step.if, diff --git a/scripts/solheim-smoke/README.md b/scripts/solheim-smoke/README.md index 30b331ae90..6ac5f30911 100644 --- a/scripts/solheim-smoke/README.md +++ b/scripts/solheim-smoke/README.md @@ -38,7 +38,10 @@ Activation and completion each have a 120-second limit; task-start acknowledgmen in the controller. The extension awaits startup and configuration persistence without racing shared-profile writes against a timer. If the controller times out, it tears down its isolated VS Code process rather than trying to roll back shared settings or remove a possibly unrelated task. The live CI step has an eight-minute outer timeout. -Teardown escalates through close, terminate, and kill within independent five-second windows. A non-empty +Teardown escalates through close, terminate, and kill within independent five-second windows. After the +completion message, the driver first holds the finished task on screen for three seconds, so the recording ends +on the completed chat before teardown clears it. The display dependency install step in the workflow is bounded +at five minutes, so a slow apt mirror fails the job instead of consuming it. A non-empty completion and actual output-token usage must belong to the accepted task. Activation alone cannot pass. The driver never approves dialogs or tool execution; it closes the task during teardown. @@ -53,7 +56,10 @@ It does not contradict the `OK` verdict. See The action uploads `verdict.json` (fixed failure code, boolean gates, duration and provider/model names), `trace.json` (versioned metadata-only structured trace from the shared substrate; see [`scripts/agent-evals/README.md`](../agent-evals/README.md) for its accounting limits), and `smoke.mp4`, -a 1280×720, 12 fps screen recording of the fresh CI virtual display. It records up to +a 1280×720, 12 fps screen recording of the fresh CI virtual display. The recording shows the Zoo Code chat +panel: `sidebar-runner.cjs` runs inside the extension host through `--extensionTestsPath`, focuses the Zoo Code +sidebar, and then keeps its `run()` pending for the whole smoke, because VS Code exits once that `run()` settles. +It records up to eight minutes / approximately 120 MiB, with bounded shutdown and seven-day artifact retention. Video contains the visible smoke UI, potentially including the fixed prompt and model answer; unlike the verdict and the trace, it is not text-redacted. No settings screen is opened, and no host logs, storage, task diff --git a/scripts/solheim-smoke/config.test.ts b/scripts/solheim-smoke/config.test.ts index 345062f0d3..d770ca6155 100644 --- a/scripts/solheim-smoke/config.test.ts +++ b/scripts/solheim-smoke/config.test.ts @@ -62,4 +62,16 @@ describe("Solheim provider smoke configuration", () => { assert.ok(args.includes("--extensionDevelopmentPath=/extension")) assert.ok(args.includes("--user-data-dir=/data")) }) + it("adds the extension tests path only when one is supplied", () => { + const plain = buildLaunchArgs("/empty-workspace", "/data", "/extensions", "/extension") + assert.ok(!plain.some((arg) => arg.startsWith("--extensionTestsPath="))) + const withTests = buildLaunchArgs( + "/empty-workspace", + "/data", + "/extensions", + "/extension", + "/runner/sidebar-runner.cjs", + ) + assert.ok(withTests.includes("--extensionTestsPath=/runner/sidebar-runner.cjs")) + }) }) diff --git a/scripts/solheim-smoke/config.ts b/scripts/solheim-smoke/config.ts index b394d12cf6..215a37d588 100644 --- a/scripts/solheim-smoke/config.ts +++ b/scripts/solheim-smoke/config.ts @@ -5,6 +5,9 @@ export { buildChildEnvironment, buildLaunchArgs } from "../agent-evals/isolation export const SOLHEIM_BASE_URL = "https://api.solheim.ai/v1" export const SOLHEIM_MODEL = "qwen3.8-27b" export const SMOKE_TIMEOUT_MS = 120_000 +// Keeps the finished task on screen before teardown clears it, so the +// recording ends on the completed chat. +export const SMOKE_TEARDOWN_HOLD_MS = 3_000 export const SMOKE_PROMPT = "This is a provider connectivity smoke test, not a code review. Call attempt_completion once with the result: Provider smoke completed. Do not read files, use other tools, delegate, or judge code." diff --git a/scripts/solheim-smoke/driver.mts b/scripts/solheim-smoke/driver.mts index df10cd5566..ce5f44482b 100644 --- a/scripts/solheim-smoke/driver.mts +++ b/scripts/solheim-smoke/driver.mts @@ -8,7 +8,17 @@ const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..") const out = process.env.SOLHEIM_SMOKE_OUT_DIR if (!out) throw new Error("Set SOLHEIM_SMOKE_OUT_DIR to a scratch directory") await fs.mkdir(out, { recursive: true }) -const runtime = createHostRuntime({ root, artifactsDir: out }) +const testsDiagnosticsPath = path.resolve(out, "sidebar-focus.json") +// Replace stale evidence even if the host never reaches the sidebar module. +await fs.writeFile(testsDiagnosticsPath, JSON.stringify({ status: "pending" })) +const runtime = createHostRuntime({ + root, + artifactsDir: out, + testsDiagnosticsPath, + // The runner module focuses the Zoo Code sidebar so the recording shows + // the chat panel. Its run() never settles; the driver owns teardown. + testsPath: path.join(root, "scripts", "solheim-smoke", "sidebar-runner.cjs"), +}) try { const verdict = await runSmoke(runtime, process.env.SOLHEIM_API_KEY ?? "") console.log(`Solheim provider smoke: ${verdict.code}`) diff --git a/scripts/solheim-smoke/run.test.ts b/scripts/solheim-smoke/run.test.ts index 9be74b3d75..d27a0bb229 100644 --- a/scripts/solheim-smoke/run.test.ts +++ b/scripts/solheim-smoke/run.test.ts @@ -1,6 +1,7 @@ import assert from "node:assert/strict" import { describe, it } from "node:test" import { RooCodeEventName, TaskCommandName, type TaskCommand } from "@roo-code/types" +import { SMOKE_TEARDOWN_HOLD_MS } from "./config.ts" import { runSmoke, smokeExitCode, type SmokeRuntime, type SmokeVerdict } from "./run.ts" type Scenario = @@ -164,6 +165,9 @@ describe("provider smoke orchestration", () => { const verdict = await runSmoke(f.runtime, "private-test-key") assert.equal(verdict.code, "OK") assert.equal(smokeExitCode(verdict), 0) + // The teardown hold runs once after the completion message; no other + // pause fires in this scenario, so the duration is exactly the hold. + assert.equal(verdict.durationMs, SMOKE_TEARDOWN_HOLD_MS) assert.deepEqual(verdict.gates, { extensionActivated: true, ipcConnected: true, @@ -210,7 +214,8 @@ describe("provider smoke orchestration", () => { const f = fixture("success", { stuck: true }) const verdict = await runSmoke(f.runtime, "private-test-key") assert.equal(verdict.code, "TEARDOWN_FAILED") - assert.equal(verdict.durationMs, 15_000) + // The 3 second completion hold plus three 5 second termination windows. + assert.equal(verdict.durationMs, 15_000 + SMOKE_TEARDOWN_HOLD_MS) assert.deepEqual(f.signals, ["SIGTERM", "SIGKILL"]) assert.equal(verdict.gates.childExited, false) assert.equal(f.cleaned, false) diff --git a/scripts/solheim-smoke/run.ts b/scripts/solheim-smoke/run.ts index 8cbf1e9fb5..b2aed5c0e9 100644 --- a/scripts/solheim-smoke/run.ts +++ b/scripts/solheim-smoke/run.ts @@ -1,6 +1,6 @@ import { runAgentFixture, type AgentRuntime } from "../agent-evals/runner.ts" import { getFixture } from "../agent-evals/registry.ts" -import { SOLHEIM_BASE_URL, SOLHEIM_MODEL } from "./config.ts" +import { SOLHEIM_BASE_URL, SOLHEIM_MODEL, SMOKE_TEARDOWN_HOLD_MS } from "./config.ts" import { collectEvidence, smokeCode, type SmokeCode, type SmokeEvidence } from "./evidence.ts" import { SMOKE_FIXTURE_NAME, ensureSmokeFixtureRegistered } from "./fixture.ts" @@ -45,6 +45,7 @@ export async function runSmoke(runtime: SmokeRuntime, apiKey: string): Promise new Promise((resolve) => setTimeout(resolve, ms)) + +async function focusSidebarWhenReady() { + const extension = vscode.extensions.getExtension(EXTENSION_ID) + if (!extension) throw new Error("Zoo Code extension not found") + const api = extension.isActive ? extension.exports : await extension.activate() + await vscode.commands.executeCommand(FOCUS_COMMAND) + const deadline = Date.now() + READY_TIMEOUT_MS + while (!api.isReady()) { + if (Date.now() >= deadline) throw new Error("Zoo Code webview did not launch in time") + await delay(POLL_INTERVAL_MS) + } +} + +async function writeStatus(status) { + const target = process.env.AGENT_EVALS_TESTS_DIAGNOSTICS_PATH + if (!target) return + // Fixed metadata only: extension errors can contain private host details. + await fs.writeFile(target, JSON.stringify({ status })) +} + +exports.run = async () => { + try { + await focusSidebarWhenReady() + await writeStatus("ready") + } catch { + // Focus is best effort for the recording. A rejected run() would end + // the test host and cut the smoke short. Persist bounded evidence instead. + await writeStatus("failed").catch(() => {}) + } + await new Promise(() => {}) +} diff --git a/scripts/solheim-smoke/sidebar-runner.test.ts b/scripts/solheim-smoke/sidebar-runner.test.ts new file mode 100644 index 0000000000..578368d372 --- /dev/null +++ b/scripts/solheim-smoke/sidebar-runner.test.ts @@ -0,0 +1,73 @@ +import assert from "node:assert/strict" +import fs from "node:fs/promises" +import { describe, it } from "node:test" +import vm from "node:vm" + +const source = await fs.readFile(new URL("./sidebar-runner.cjs", import.meta.url), "utf8") + +describe("sidebar recording diagnostics", () => { + for (const scenario of ["ready", "focus-failed", "extension-missing", "write-failed"] as const) { + it(`persists bounded evidence and keeps the host pending for ${scenario}`, async () => { + const writes: { target: string; body: string }[] = [] + const commands: string[] = [] + const exports: { run?: () => Promise } = {} + const context = vm.createContext({ + exports, + process: { env: { AGENT_EVALS_TESTS_DIAGNOSTICS_PATH: "/artifacts/sidebar-focus.json" } }, + require(name: string) { + if (name === "node:fs/promises") + return { + async writeFile(target: string, body: string) { + if (scenario === "write-failed") throw new Error("private filesystem detail") + writes.push({ target, body }) + }, + } + assert.equal(name, "vscode") + return { + extensions: { + getExtension: () => + scenario === "extension-missing" + ? undefined + : { + isActive: true, + exports: { isReady: () => true }, + }, + }, + commands: { + async executeCommand(command: string) { + commands.push(command) + if (scenario === "focus-failed") throw new Error("private activation detail") + }, + }, + } + }, + }) + vm.runInContext(source, context) + assert.ok(exports.run) + let settled = false + void exports.run().then( + () => { + settled = true + }, + () => { + settled = true + }, + ) + // Flush async focus and file writes without waiting for the intentionally pending run(). + await new Promise((resolve) => setImmediate(resolve)) + assert.equal(settled, false) + assert.deepEqual(commands, scenario === "extension-missing" ? [] : ["zoo-code.SidebarProvider.focus"]) + assert.deepEqual( + writes, + scenario === "write-failed" + ? [] + : [ + { + target: "/artifacts/sidebar-focus.json", + body: JSON.stringify({ status: scenario === "ready" ? "ready" : "failed" }), + }, + ], + ) + }) + } +})