[Chore] Show the Zoo Code panel in the Solheim smoke video and bound the apt step

.github/workflows/solheim-provider-smoke.yml

@@ -41,6 +41,7 @@ jobs:
                   pnpm --filter @roo-code/vscode-webview build
 
             - name: Install display dependencies
+              timeout-minutes: 5
               run: sudo apt-get update && sudo apt-get install -y xvfb ffmpeg libasound2t64
 
             - name: Run Solheim provider smoke
@@ -73,6 +74,7 @@ jobs:
                   path: |
                       ${{ runner.temp }}/solheim-provider-smoke/verdict.json
                       ${{ runner.temp }}/solheim-provider-smoke/trace.json
+                      ${{ runner.temp }}/solheim-provider-smoke/sidebar-focus.json
                   if-no-files-found: error
                   retention-days: 7
 

scripts/agent-evals/README.md

@@ -30,6 +30,11 @@ Execution limits are enforced, not decorative. The controller applies the
 activation, task start, task run, usage drain, and teardown step limits, and
 the trace records the measured durations.
 
+Two launch options are off by default. `testsPath` adds `--extensionTestsPath`
+to the VS Code launch; that module's `run()` must never settle, because VS Code
+exits once it does. `holdBeforeTeardownMs` holds the finished task on screen
+for that window after the completion message, before teardown starts.
+
 ## Structured trace
 
 Each run writes `trace.json` with `traceSchemaVersion: 1`. The trace is

scripts/agent-evals/isolation.ts

@@ -19,12 +19,21 @@ export function buildChildEnvironment(
 	}
 }
 
-export function buildLaunchArgs(workspace: string, userData: string, extensions: string, extension: string): string[] {
+export function buildLaunchArgs(
+	workspace: string,
+	userData: string,
+	extensions: string,
+	extension: string,
+	testsPath?: string,
+): string[] {
 	return [
 		workspace,
 		`--user-data-dir=${userData}`,
 		`--extensions-dir=${extensions}`,
 		`--extensionDevelopmentPath=${extension}`,
+		// Off by default: an --extensionTestsPath module settles into a VS Code
+		// exit, so only a caller that owns a never-settling module may set it.
+		...(testsPath ? [`--extensionTestsPath=${testsPath}`] : []),
 		"--no-sandbox",
 		"--disable-gpu-sandbox",
 		"--password-store=basic",

scripts/agent-evals/launcher.ts

@@ -17,6 +17,11 @@ export interface HostRuntimeOptions {
 	artifactsDir: string
 	now?: () => number
 	pause?: (ms: number) => Promise<void>
+	// Optional --extensionTestsPath module. The module's run() must never
+	// settle, because VS Code exits when it does. Off by default.
+	testsPath?: string
+	// Optional bounded status artifact written by the extension test module.
+	testsDiagnosticsPath?: string
 }
 
 const readCommit = async (root: string): Promise<MeasuredString> => {
@@ -84,13 +89,19 @@ export function createHostRuntime(options: HostRuntimeOptions): AgentRuntime {
 				path.join(temp, "userdata"),
 				path.join(temp, "extensions"),
 				extension,
+				options.testsPath,
 			)
 			const child = spawn(
 				process.env.DISPLAY ? executable : "xvfb-run",
 				process.env.DISPLAY ? args : ["-a", executable, ...args],
 				{
 					detached: true,
-					env: buildChildEnvironment(process.env, isolatedHome, socket),
+					env: {
+						...buildChildEnvironment(process.env, isolatedHome, socket),
+						...(options.testsDiagnosticsPath
+							? { AGENT_EVALS_TESTS_DIAGNOSTICS_PATH: options.testsDiagnosticsPath }
+							: {}),
+					},
 					stdio: "ignore",
 				},
 			)

scripts/agent-evals/runner.test.ts

@@ -252,6 +252,7 @@ function harness(
 		signals,
 		artifacts,
 		emit,
+		close,
 		interrupt: () => interrupt(),
 		get cleaned() {
 			return cleaned
@@ -1050,4 +1051,94 @@ describe("agent fixture controller", () => {
 		assert.equal(result.execution.sessionLost, false)
 		assert.equal(result.teardown.status, "ok")
 	})
+
+	for (const hostExited of [false, true]) {
+		it(`holds teardown only while the completed host is live (exited=${hostExited})`, async () => {
+			const fixtureName = `controller-hold-${hostExited}`
+			registerTestFixture(fixtureName)
+			const f = harness("no-events")
+			f.emit({
+				eventName: RooCodeEventName.Message,
+				payload: [
+					{
+						taskId: "root",
+						action: "created",
+						message: { ts: 1, type: "say", say: "completion_result", text: "synthetic final answer" },
+					},
+				],
+			})
+			f.emit({
+				eventName: RooCodeEventName.TaskCompleted,
+				payload: [
+					"root",
+					{ totalTokensIn: 7, totalTokensOut: 3, totalCost: 0, contextTokens: 9 },
+					{},
+					{
+						isSubtask: false,
+					},
+				],
+			})
+			const holdPauses: number[] = []
+			let closeTaskSeenDuringHold = false
+			const pause = f.runtime.pause
+			f.runtime.pause = async (ms) => {
+				if (ms === 3_000) {
+					holdPauses.push(ms)
+					closeTaskSeenDuringHold = f.commands.some(
+						(command) => command.commandName === TaskCommandName.CloseTask,
+					)
+				}
+				await pause(ms)
+			}
+			const result = await runAgentFixture(f.runtime, {
+				fixtureName,
+				credential: CREDENTIAL,
+				observers: {
+					onAcceptedEvent: (event) => {
+						if (hostExited && event.eventName === RooCodeEventName.TaskCompleted) f.close()
+					},
+					shouldStop: () => true,
+				},
+				holdBeforeTeardownMs: 3_000,
+			})
+			assert.equal(result.execution.code, "COMPLETED")
+			assert.equal(result.execution.completionMessageSeen, true)
+			assert.deepEqual(holdPauses, hostExited ? [] : [3_000])
+			// The hold runs before teardown, so CloseTask is sent only after it.
+			assert.equal(closeTaskSeenDuringHold, false)
+			assert.equal(f.commands.at(-1)?.commandName, TaskCommandName.CloseTask)
+			assert.equal(result.teardown.status, "ok")
+			assert.equal(result.teardown.hostExited, true)
+			const trace = writtenTrace(f)
+			// CloseTask exits immediately, so none of the recording hold belongs to teardown.
+			assert.equal(trace.execution.timings.teardownMs, 0)
+		})
+	}
+	it("skips the teardown hold without a completion message", async () => {
+		registerTestFixture("controller-hold-skipped", {
+			limits: {
+				activationTimeoutMs: 120_000,
+				taskStartTimeoutMs: 35_000,
+				taskRunTimeoutMs: 200,
+				usageDrainMs: 100,
+				teardownStepMs: 100,
+			},
+		})
+		const f = harness("no-events")
+		const holdPauses: number[] = []
+		const pause = f.runtime.pause
+		f.runtime.pause = async (ms) => {
+			if (ms === 3_000) holdPauses.push(ms)
+			await pause(ms)
+		}
+		const result = await runAgentFixture(f.runtime, {
+			fixtureName: "controller-hold-skipped",
+			credential: CREDENTIAL,
+			holdBeforeTeardownMs: 3_000,
+		})
+		assert.equal(result.execution.code, "TASK_RUN_TIMEOUT")
+		assert.equal(result.execution.completionMessageSeen, false)
+		assert.deepEqual(holdPauses, [])
+		assert.equal(result.teardown.status, "ok")
+	})
 })

scripts/agent-evals/runner.ts

@@ -57,6 +57,10 @@ export interface AgentRunOptions {
 	fixtureName: string
 	credential: string
 	observers?: AgentRunObservers
+	// Optional hold before teardown after the completion message, so a
+	// recording can capture the finished task. Off by default; skipped when
+	// no completion message was seen or the host is already gone.
+	holdBeforeTeardownMs?: number
 }
 
 export interface AgentRunResult {
@@ -325,6 +329,11 @@ export async function runAgentFixture(runtime: AgentRuntime, options: AgentRunOp
 		// Never expose thrown provider, filesystem or host messages.
 	} finally {
 		waiter?.onExit()
+		// A recording needs the finished task on screen before teardown
+		// clears it. The hold is opt-in and only applies after the
+		// completion message, while a live host can still show it.
+		if (completionMessageSeen && options.holdBeforeTeardownMs && host && !host.exited)
+			await runtime.pause(options.holdBeforeTeardownMs)
 		const teardownStart = runtime.now()
 		const attempt = async (step: string, operation: () => unknown) => {
 			try {

scripts/solheim-smoke-workflow.test.mjs

@@ -34,6 +34,12 @@ describe("Solheim provider smoke workflow policy", () => {
 		assert.equal(checkouts[0].with["persist-credentials"], false)
 	})
 
+	it("bounds the display dependency install to five minutes", () => {
+		const apt = steps.find((step) => step.name === "Install display dependencies")
+		assert.ok(apt, "missing display dependency step")
+		assert.equal(apt["timeout-minutes"], 5)
+	})
+
 	it("exposes the provider credential only to the bounded provider step", () => {
 		const provider = steps.find((step) => step.id === "provider")
 		assert.ok(provider, "missing provider step")
@@ -61,7 +67,10 @@ describe("Solheim provider smoke workflow policy", () => {
 				.split("\n")
 				.map((line) => line.trim()),
 		)
-		assert.deepEqual(paths, [[output + "/verdict.json", output + "/trace.json"], [output + "/smoke.mp4"]])
+		assert.deepEqual(paths, [
+			[output + "/verdict.json", output + "/trace.json", output + "/sidebar-focus.json"],
+			[output + "/smoke.mp4"],
+		])
 		for (const step of uploads) {
 			assert.equal(
 				step.if,

scripts/solheim-smoke/README.md

@@ -38,7 +38,10 @@ Activation and completion each have a 120-second limit; task-start acknowledgmen
 in the controller. The extension awaits startup and configuration persistence without racing shared-profile
 writes against a timer. If the controller times out, it tears down its isolated VS Code process rather than
 trying to roll back shared settings or remove a possibly unrelated task. The live CI step has an eight-minute outer timeout.
-Teardown escalates through close, terminate, and kill within independent five-second windows. A non-empty
+Teardown escalates through close, terminate, and kill within independent five-second windows. After the
+completion message, the driver first holds the finished task on screen for three seconds, so the recording ends
+on the completed chat before teardown clears it. The display dependency install step in the workflow is bounded
+at five minutes, so a slow apt mirror fails the job instead of consuming it. A non-empty
 completion and actual output-token usage must belong to the accepted task. Activation alone cannot pass.
 The driver never approves dialogs or tool execution; it closes the task during teardown.
 
@@ -53,7 +56,10 @@ It does not contradict the `OK` verdict. See
 The action uploads `verdict.json` (fixed failure code, boolean gates, duration and provider/model names),
 `trace.json` (versioned metadata-only structured trace from the shared substrate; see
 [`scripts/agent-evals/README.md`](../agent-evals/README.md) for its accounting limits), and `smoke.mp4`,
-a 1280×720, 12 fps screen recording of the fresh CI virtual display. It records up to
+a 1280×720, 12 fps screen recording of the fresh CI virtual display. The recording shows the Zoo Code chat
+panel: `sidebar-runner.cjs` runs inside the extension host through `--extensionTestsPath`, focuses the Zoo Code
+sidebar, and then keeps its `run()` pending for the whole smoke, because VS Code exits once that `run()` settles.
+It records up to
 eight minutes / approximately 120 MiB, with bounded shutdown and seven-day artifact retention. Video
 contains the visible smoke UI, potentially including the fixed prompt and model answer; unlike the verdict
 and the trace, it is not text-redacted. No settings screen is opened, and no host logs, storage, task

scripts/solheim-smoke/config.test.ts

@@ -62,4 +62,16 @@ describe("Solheim provider smoke configuration", () => {
 		assert.ok(args.includes("--extensionDevelopmentPath=/extension"))
 		assert.ok(args.includes("--user-data-dir=/data"))
 	})
+	it("adds the extension tests path only when one is supplied", () => {
+		const plain = buildLaunchArgs("/empty-workspace", "/data", "/extensions", "/extension")
+		assert.ok(!plain.some((arg) => arg.startsWith("--extensionTestsPath=")))
+		const withTests = buildLaunchArgs(
+			"/empty-workspace",
+			"/data",
+			"/extensions",
+			"/extension",
+			"/runner/sidebar-runner.cjs",
+		)
+		assert.ok(withTests.includes("--extensionTestsPath=/runner/sidebar-runner.cjs"))
+	})
 })

scripts/solheim-smoke/config.ts

@@ -5,6 +5,9 @@ export { buildChildEnvironment, buildLaunchArgs } from "../agent-evals/isolation
 export const SOLHEIM_BASE_URL = "https://api.solheim.ai/v1"
 export const SOLHEIM_MODEL = "qwen3.8-27b"
 export const SMOKE_TIMEOUT_MS = 120_000
+// Keeps the finished task on screen before teardown clears it, so the
+// recording ends on the completed chat.
+export const SMOKE_TEARDOWN_HOLD_MS = 3_000
 export const SMOKE_PROMPT =
 	"This is a provider connectivity smoke test, not a code review. Call attempt_completion once with the result: Provider smoke completed. Do not read files, use other tools, delegate, or judge code."
 

scripts/solheim-smoke/driver.mts

@@ -8,7 +8,17 @@ const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..")
 const out = process.env.SOLHEIM_SMOKE_OUT_DIR
 if (!out) throw new Error("Set SOLHEIM_SMOKE_OUT_DIR to a scratch directory")
 await fs.mkdir(out, { recursive: true })
-const runtime = createHostRuntime({ root, artifactsDir: out })
+const testsDiagnosticsPath = path.resolve(out, "sidebar-focus.json")
+// Replace stale evidence even if the host never reaches the sidebar module.
+await fs.writeFile(testsDiagnosticsPath, JSON.stringify({ status: "pending" }))
+const runtime = createHostRuntime({
+	root,
+	artifactsDir: out,
+	testsDiagnosticsPath,
+	// The runner module focuses the Zoo Code sidebar so the recording shows
+	// the chat panel. Its run() never settles; the driver owns teardown.
+	testsPath: path.join(root, "scripts", "solheim-smoke", "sidebar-runner.cjs"),
+})
 try {
 	const verdict = await runSmoke(runtime, process.env.SOLHEIM_API_KEY ?? "")
 	console.log(`Solheim provider smoke: ${verdict.code}`)

scripts/solheim-smoke/run.test.ts

@@ -1,6 +1,7 @@
 import assert from "node:assert/strict"
 import { describe, it } from "node:test"
 import { RooCodeEventName, TaskCommandName, type TaskCommand } from "@roo-code/types"
+import { SMOKE_TEARDOWN_HOLD_MS } from "./config.ts"
 import { runSmoke, smokeExitCode, type SmokeRuntime, type SmokeVerdict } from "./run.ts"
 
 type Scenario =
@@ -164,6 +165,9 @@ describe("provider smoke orchestration", () => {
 		const verdict = await runSmoke(f.runtime, "private-test-key")
 		assert.equal(verdict.code, "OK")
 		assert.equal(smokeExitCode(verdict), 0)
+		// The teardown hold runs once after the completion message; no other
+		// pause fires in this scenario, so the duration is exactly the hold.
+		assert.equal(verdict.durationMs, SMOKE_TEARDOWN_HOLD_MS)
 		assert.deepEqual(verdict.gates, {
 			extensionActivated: true,
 			ipcConnected: true,
@@ -210,7 +214,8 @@ describe("provider smoke orchestration", () => {
 		const f = fixture("success", { stuck: true })
 		const verdict = await runSmoke(f.runtime, "private-test-key")
 		assert.equal(verdict.code, "TEARDOWN_FAILED")
-		assert.equal(verdict.durationMs, 15_000)
+		// The 3 second completion hold plus three 5 second termination windows.
+		assert.equal(verdict.durationMs, 15_000 + SMOKE_TEARDOWN_HOLD_MS)
 		assert.deepEqual(f.signals, ["SIGTERM", "SIGKILL"])
 		assert.equal(verdict.gates.childExited, false)
 		assert.equal(f.cleaned, false)

scripts/solheim-smoke/run.ts

@@ -1,6 +1,6 @@
 import { runAgentFixture, type AgentRuntime } from "../agent-evals/runner.ts"
 import { getFixture } from "../agent-evals/registry.ts"
-import { SOLHEIM_BASE_URL, SOLHEIM_MODEL } from "./config.ts"
+import { SOLHEIM_BASE_URL, SOLHEIM_MODEL, SMOKE_TEARDOWN_HOLD_MS } from "./config.ts"
 import { collectEvidence, smokeCode, type SmokeCode, type SmokeEvidence } from "./evidence.ts"
 import { SMOKE_FIXTURE_NAME, ensureSmokeFixtureRegistered } from "./fixture.ts"
 
@@ -45,6 +45,7 @@ export async function runSmoke(runtime: SmokeRuntime, apiKey: string): Promise<S
 				evidence.aborted ||
 				(evidence.providerResponseSeen && evidence.completionSeen),
 		},
+		holdBeforeTeardownMs: SMOKE_TEARDOWN_HOLD_MS,
 	})
 	let code: SmokeCode
 	switch (result.execution.code) {

scripts/solheim-smoke/sidebar-runner.cjs

@@ -0,0 +1,48 @@
+"use strict"
+
+// Runs inside the extension host through --extensionTestsPath so the smoke
+// recording shows the Zoo Code chat panel. VS Code exits once the run()
+// export settles, so run() must never settle: it focuses the sidebar, waits
+// for the webview, then stays pending until the driver tears the host down.
+// The IPC start path keeps focusSidebar: false; this module owns focus instead.
+
+const vscode = require("vscode")
+const fs = require("node:fs/promises")
+
+const EXTENSION_ID = "ZooCodeOrganization.zoo-code"
+const FOCUS_COMMAND = "zoo-code.SidebarProvider.focus"
+const POLL_INTERVAL_MS = 100
+const READY_TIMEOUT_MS = 120_000
+
+const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms))
+
+async function focusSidebarWhenReady() {
+	const extension = vscode.extensions.getExtension(EXTENSION_ID)
+	if (!extension) throw new Error("Zoo Code extension not found")
+	const api = extension.isActive ? extension.exports : await extension.activate()
+	await vscode.commands.executeCommand(FOCUS_COMMAND)
+	const deadline = Date.now() + READY_TIMEOUT_MS
+	while (!api.isReady()) {
+		if (Date.now() >= deadline) throw new Error("Zoo Code webview did not launch in time")
+		await delay(POLL_INTERVAL_MS)
+	}
+}
+
+async function writeStatus(status) {
+	const target = process.env.AGENT_EVALS_TESTS_DIAGNOSTICS_PATH
+	if (!target) return
+	// Fixed metadata only: extension errors can contain private host details.
+	await fs.writeFile(target, JSON.stringify({ status }))
+}
+
+exports.run = async () => {
+	try {
+		await focusSidebarWhenReady()
+		await writeStatus("ready")
+	} catch {
+		// Focus is best effort for the recording. A rejected run() would end
+		// the test host and cut the smoke short. Persist bounded evidence instead.
+		await writeStatus("failed").catch(() => {})
+	}
+	await new Promise(() => {})
+}