diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 84dcadb1fa..5a3fdcdb29 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,7 +42,7 @@ jobs: fi - name: Test CI planner - run: node --test --test-concurrency=1 scripts/ci-test-plan.test.mjs + run: node --test --test-concurrency=1 scripts/ci-test-plan.test.mjs scripts/verify-windows-harness.test.mjs # Same shape and the same needs: a regenerate-and-diff contract that runs # on Node alone, so it belongs beside the planner test rather than behind diff --git a/scripts/verify-packaged-app.mjs b/scripts/verify-packaged-app.mjs index a8b6daa235..eed1fbaa38 100644 --- a/scripts/verify-packaged-app.mjs +++ b/scripts/verify-packaged-app.mjs @@ -1,8 +1,7 @@ import { spawn } from 'node:child_process'; import { createHash } from 'node:crypto'; import { createReadStream } from 'node:fs'; -import { access, mkdir } from 'node:fs/promises'; -import { createServer } from 'node:net'; +import { access, mkdir, readdir } from 'node:fs/promises'; import { join } from 'node:path'; // `timeoutMs` is opt-in, for the commands that have actually hung: node-pty @@ -74,21 +73,52 @@ export async function assertMissing(path) { throw new Error(`Forbidden release resource exists: ${path}`); } -export async function reserveTcpPort() { - const server = createServer(); - await new Promise((resolvePromise, reject) => { - server.once('error', reject); - server.listen(0, '127.0.0.1', resolvePromise); - }); - const address = server.address(); - if (!address || typeof address === 'string') { - server.close(); - throw new Error('Could not reserve a CDP port.'); - } - await new Promise((resolvePromise, reject) => { - server.close((error) => (error ? reject(error) : resolvePromise())); +/** + * The authoritative CDP port: Chromium announces it on stderr once the + * DevTools socket is actually bound. Callers spawn with + * `--remote-debugging-port=0` and wait for this instead of pre-reserving a + * port — reserve-then-release had a race window in which another process + * could take the port, leaving Electron listening elsewhere while the + * verifier polled the stale number for its full deadline ("did not expose + * CDP ... fetch failed", observed repeatedly on busy CI runners). + */ +export function waitForDevToolsPort(child, { timeoutMs = 30_000 } = {}) { + return new Promise((resolvePromise, reject) => { + let buffer = ''; + const cleanup = () => { + clearTimeout(timeout); + child.stderr.off('data', onData); + child.off('exit', onExit); + }; + const timeout = setTimeout(() => { + cleanup(); + reject( + new Error( + `Packaged Maka did not announce a DevTools port within ${timeoutMs}ms.` + + `${buffer.trim() ? `\n${buffer.trim()}` : ''}`, + ), + ); + }, timeoutMs); + const onData = (chunk) => { + buffer = `${buffer}${chunk}`.slice(-16_384); + const match = /DevTools listening on ws:\/\/127\.0\.0\.1:(\d+)\//.exec(buffer); + if (match) { + cleanup(); + resolvePromise(Number(match[1])); + } + }; + const onExit = () => { + cleanup(); + reject( + new Error( + `Packaged Maka exited before announcing a DevTools port.` + + `${buffer.trim() ? `\n${buffer.trim()}` : ''}`, + ), + ); + }; + child.stderr.on('data', onData); + child.once('exit', onExit); }); - return address.port; } function delay(milliseconds) { @@ -97,15 +127,24 @@ function delay(milliseconds) { }); } -export async function findRendererTarget(port, child) { - const deadline = Date.now() + 30_000; +// The default deadline is generous on purpose: windows-2025 runners have shown +// first-page creation taking beyond 30 seconds when a smoke follows multiple +// installs in the same job, and a too-tight deadline fails a good build. The +// wait is still bounded and fail-closed; a dead child short-circuits it. +export async function findRendererTarget(port, child, { timeoutMs = 90_000 } = {}) { + const deadline = Date.now() + timeoutMs; let lastError; while (Date.now() < deadline) { if (child.exitCode !== null) { throw new Error(`Packaged Maka exited before its renderer was ready.`); } try { - const response = await fetch(`http://127.0.0.1:${port}/json/list`); + // A connect that hangs (half-open or filtered socket) would otherwise + // run into the OS connect timeout and overshoot the stated deadline by + // minutes — observed as a ~6-minute "90 seconds" failure on CI. + const response = await fetch(`http://127.0.0.1:${port}/json/list`, { + signal: AbortSignal.timeout(2_000), + }); if (response.ok) { const targets = await response.json(); const page = targets.find( @@ -118,9 +157,20 @@ export async function findRendererTarget(port, child) { } await delay(250); } + // `fetch failed` alone says nothing; the cause chain carries the socket + // errno (ECONNREFUSED vs ETIMEDOUT vs ECONNRESET), which is the evidence + // that distinguishes "DevTools never listened" from "something filtered it". + const described = []; + for (let error = lastError; error; error = error.cause) { + if (Array.isArray(error.errors) && error.errors.length) { + described.push(error.errors.map((inner) => inner.message ?? String(inner)).join(' & ')); + } else { + described.push(error.message ?? String(error)); + } + } throw new Error( - `Packaged Maka renderer did not expose CDP within 30 seconds${ - lastError ? `: ${lastError.message}` : '' + `Packaged Maka renderer did not expose CDP within ${Math.round(timeoutMs / 1000)} seconds${ + described.length ? `: ${described.join(' <- ')}` : '' }.`, ); } @@ -140,10 +190,36 @@ export async function evaluateInRenderer( throw new Error('The release verifier requires Node.js WebSocket support.'); } const socket = new WebSocket(webSocketDebuggerUrl); - await new Promise((resolvePromise, reject) => { - socket.addEventListener('open', resolvePromise, { once: true }); - socket.addEventListener('error', reject, { once: true }); - }); + try { + // The handshake needs its own bound: a DevTools port that accepts TCP + // but never speaks raises neither `open` nor `error`, and an unbounded + // await here would make every retry loop built on this helper hang to + // the workflow timeout instead of failing one probe. + await new Promise((resolvePromise, reject) => { + const timeout = setTimeout(() => { + reject(new Error(`CDP WebSocket did not open within ${timeoutMs}ms.`)); + }, timeoutMs); + socket.addEventListener( + 'open', + () => { + clearTimeout(timeout); + resolvePromise(); + }, + { once: true }, + ); + socket.addEventListener( + 'error', + (event) => { + clearTimeout(timeout); + reject(event.error ?? new Error('CDP WebSocket connection failed.')); + }, + { once: true }, + ); + }); + } catch (error) { + socket.close(); + throw error; + } try { return await new Promise((resolvePromise, reject) => { @@ -195,8 +271,8 @@ export const RENDERER_STATE_EXPRESSION = `({ hasAppShell: Boolean(document.querySelector('#root [data-agents-page]')) })`; -function evaluateRenderer(webSocketDebuggerUrl) { - return evaluateInRenderer(webSocketDebuggerUrl, RENDERER_STATE_EXPRESSION); +function evaluateRenderer(webSocketDebuggerUrl, timeoutMs) { + return evaluateInRenderer(webSocketDebuggerUrl, RENDERER_STATE_EXPRESSION, { timeoutMs }); } export function isPackagedRendererUsable(rendererState) { @@ -209,6 +285,48 @@ export function isPackagedRendererUsable(rendererState) { ); } +/** + * Poll a freshly booted packaged app over CDP until its renderer reports the + * usable state. One evaluation can stall past its own socket timeout while + * the renderer is still booting — observed on the Windows release runners, + * where a single timed-out `Runtime.evaluate` used to fail the whole gate. + * The deadline here is the authority: an individual failed probe is retried, + * not fatal, and only the deadline (or child exit) fails the wait. The last + * probe error or renderer state is reported as evidence either way. + */ +export async function waitForUsableRenderer( + webSocketDebuggerUrl, + child, + { deadlineMs = 30_000, description = 'Packaged renderer' } = {}, +) { + const deadline = Date.now() + deadlineMs; + let state; + let lastError; + for (;;) { + try { + state = await evaluateRenderer( + webSocketDebuggerUrl, + Math.max(1, Math.min(10_000, deadline - Date.now())), + ); + lastError = undefined; + if (isPackagedRendererUsable(state)) return; + } catch (error) { + lastError = error; + } + if (child.exitCode !== null) { + throw new Error(`${description} exited before it became usable.`); + } + if (Date.now() >= deadline) { + throw new Error( + `${description} did not become usable within ${deadlineMs}ms: ${ + lastError ? lastError.message : JSON.stringify(state) + }`, + ); + } + await delay(250); + } +} + export async function stopChild(child) { if (child.exitCode !== null) return; child.kill('SIGTERM'); @@ -271,7 +389,6 @@ export function isolatedUserEnv(homeDirectory, { temporaryDirectory = homeDirect } export async function smokePackagedRenderer(executable, { workingDirectory } = {}) { - const port = await reserveTcpPort(); const home = join(workingDirectory, 'home'); const userData = join(workingDirectory, 'user-data'); const userEnv = isolatedUserEnv(home); @@ -281,7 +398,7 @@ export async function smokePackagedRenderer(executable, { workingDirectory } = { await mkdir(userEnv.LOCALAPPDATA, { recursive: true }); const child = spawn( executable, - [`--remote-debugging-port=${port}`, `--user-data-dir=${userData}`, '--enable-logging=stderr'], + ['--remote-debugging-port=0', `--user-data-dir=${userData}`, '--enable-logging=stderr'], { cwd: workingDirectory, env: { @@ -299,20 +416,9 @@ export async function smokePackagedRenderer(executable, { workingDirectory } = { }); try { + const port = await waitForDevToolsPort(child); const target = await findRendererTarget(port, child); - const deadline = Date.now() + 30_000; - let rendererState; - while (Date.now() < deadline) { - rendererState = await evaluateRenderer(target.webSocketDebuggerUrl); - if (isPackagedRendererUsable(rendererState)) { - return; - } - if (child.exitCode !== null) { - throw new Error('Packaged Maka exited before React mounted.'); - } - await delay(250); - } - throw new Error(`Packaged renderer did not become usable: ${JSON.stringify(rendererState)}`); + await waitForUsableRenderer(target.webSocketDebuggerUrl, child); } catch (error) { throw new Error(`${error.message}${stderr.trim() ? `\n${stderr.trim()}` : ''}`); } finally { @@ -386,6 +492,54 @@ export async function assertPackagedResources( } } +/** + * Recursive content manifest of a directory tree: POSIX-normalized relative + * paths, sorted, each with its file's SHA-256. Nothing is skipped — an install + * tree has no entries whose drift would be acceptable — and anything that is + * not a plain file or directory (symlinks, junctions, devices) throws: an + * install tree must not contain them, and silently hashing a link target would + * make two different trees compare equal. + */ +export async function directoryTreeManifest(rootDirectory) { + const entries = []; + const walk = async (directory, prefix) => { + const children = await readdir(directory, { withFileTypes: true }); + // Empty directories are recorded (trailing slash, null hash) so a + // restore that loses one shows up as `missing` — files alone cannot + // witness an empty directory. + if (children.length === 0 && prefix !== '') { + entries.push({ path: `${prefix}/`, sha256: null }); + return; + } + for (const child of children) { + const absolute = join(directory, child.name); + const relative = prefix === '' ? child.name : `${prefix}/${child.name}`; + if (child.isDirectory()) { + await walk(absolute, relative); + } else if (child.isFile()) { + entries.push({ path: relative, sha256: await sha256File(absolute) }); + } else { + throw new Error(`Unsupported directory entry in ${rootDirectory}: ${relative}`); + } + } + }; + await walk(rootDirectory, ''); + entries.sort((left, right) => (left.path < right.path ? -1 : left.path > right.path ? 1 : 0)); + return entries; +} + +/** Difference between two directoryTreeManifest results, keyed by path. */ +export function diffTreeManifests(before, after) { + const beforeByPath = new Map(before.map((entry) => [entry.path, entry.sha256])); + const afterByPath = new Map(after.map((entry) => [entry.path, entry.sha256])); + const missing = before.filter((entry) => !afterByPath.has(entry.path)).map((entry) => entry.path); + const extra = after.filter((entry) => !beforeByPath.has(entry.path)).map((entry) => entry.path); + const changed = before + .filter((entry) => afterByPath.has(entry.path) && afterByPath.get(entry.path) !== entry.sha256) + .map((entry) => entry.path); + return { missing, extra, changed }; +} + export async function sha256File(path) { const hash = createHash('sha256'); const file = createReadStream(path); diff --git a/scripts/verify-windows-autoupdate.mjs b/scripts/verify-windows-autoupdate.mjs index 21dbe836ba..0a02aceef1 100644 --- a/scripts/verify-windows-autoupdate.mjs +++ b/scripts/verify-windows-autoupdate.mjs @@ -1,5 +1,6 @@ import { spawn } from 'node:child_process'; -import { access, mkdir, mkdtemp, readFile, rm } from 'node:fs/promises'; +import { existsSync } from 'node:fs'; +import { access, mkdir, mkdtemp, readFile, rm, stat } from 'node:fs/promises'; import { createServer } from 'node:http'; import { tmpdir } from 'node:os'; import { basename, join, resolve } from 'node:path'; @@ -7,18 +8,19 @@ import { pathToFileURL } from 'node:url'; import { evaluateInRenderer, findRendererTarget, - isPackagedRendererUsable, isolatedUserEnv, - RENDERER_STATE_EXPRESSION, - reserveTcpPort, + waitForDevToolsPort, + waitForUsableRenderer, runCommand, stopChild, } from './verify-packaged-app.mjs'; import { + completeInstalledApplicationUninstall, installerVersion, - listInstalledProcesses, + readUninstallDisplayVersions, + terminateInstalledProcesses, + waitForInstalledProcessAppearance, waitForInstalledProcessesToExit, - waitUntilMissing, } from './verify-windows-installer-lifecycle.mjs'; import { assertWindowsProductVersion, @@ -144,10 +146,38 @@ async function startFeedServer(files) { }; } -async function readInstalledProductVersion(executablePath, { run = runCommand } = {}) { +export async function waitForInstalledProductVersion( + executablePath, + { + run = runCommand, + timeoutMs = 60_000, + probeTimeoutMs = 10_000, + pollIntervalMs = 1_000, + sleep = delay, + } = {}, +) { const script = `(Get-Item ${powerShellLiteral(executablePath)}).VersionInfo.ProductVersion`; - const { stdout } = await run('powershell', ['-NoProfile', '-NonInteractive', '-Command', script]); - return stdout; + const deadline = Date.now() + timeoutMs; + let lastProbeError; + for (;;) { + try { + const { stdout } = await run( + 'powershell', + ['-NoProfile', '-NonInteractive', '-Command', script], + { timeoutMs: Math.max(1, Math.min(probeTimeoutMs, deadline - Date.now())) }, + ); + return stdout; + } catch (error) { + lastProbeError = error; + } + if (Date.now() >= deadline) { + throw new Error( + `Could not read the installed ProductVersion within ${timeoutMs}ms: ${lastProbeError.message}`, + { cause: lastProbeError }, + ); + } + await sleep(pollIntervalMs); + } } /** @@ -230,7 +260,6 @@ export async function verifyWindowsAutoupdate( await access(uninstaller); step('launching the installed candidate against the loopback feed'); - const cdpPort = await reserveTcpPort(); const home = join(temporaryDirectory, 'home'); const userData = join(temporaryDirectory, 'user-data'); const userEnv = isolatedUserEnv(home); @@ -250,11 +279,7 @@ export async function verifyWindowsAutoupdate( delete childEnv.MAKA_UPDATE_MOCK_STATE; child = spawn( installedExecutable, - [ - `--remote-debugging-port=${cdpPort}`, - `--user-data-dir=${userData}`, - '--enable-logging=stderr', - ], + ['--remote-debugging-port=0', `--user-data-dir=${userData}`, '--enable-logging=stderr'], { cwd: temporaryDirectory, env: childEnv, stdio: ['ignore', 'ignore', 'pipe'] }, ); let stderr = ''; @@ -263,22 +288,24 @@ export async function verifyWindowsAutoupdate( stderr = `${stderr}${chunk}`.slice(-16_384); }); - const target = await findRendererTarget(cdpPort, child); - const rendererDeadline = Date.now() + 30_000; - for (;;) { - const state = await evaluateInRenderer( - target.webSocketDebuggerUrl, - RENDERER_STATE_EXPRESSION, + const cdpPort = await waitForDevToolsPort(child); + // On a CDP attach failure the app's own stderr tail is the only evidence + // of what the packaged process was doing (renderer crash, GPU fallback, + // profile lock); the bare "fetch failed" without it has already burned + // full CI cycles on this exact stage. + const target = await findRendererTarget(cdpPort, child).catch((error) => { + throw new Error( + `${error.message}${stderr.trim() ? `\napp stderr tail:\n${stderr.trim()}` : ''}`, + { cause: error }, ); - if (isPackagedRendererUsable(state)) break; - if (child.exitCode !== null) { - throw new Error(`Candidate exited before its renderer became usable.\n${stderr.trim()}`); - } - if (Date.now() >= rendererDeadline) { - throw new Error(`Candidate renderer did not become usable: ${JSON.stringify(state)}`); - } - await delay(250); - } + }); + await waitForUsableRenderer(target.webSocketDebuggerUrl, child, { + description: 'Candidate renderer', + }).catch((error) => { + throw new Error(`${error.message}${stderr.trim() ? `\n${stderr.trim()}` : ''}`, { + cause: error, + }); + }); step('driving an update check through the renderer bridge'); await evaluateInRenderer(target.webSocketDebuggerUrl, 'window.maka.app.checkForUpdates()', { @@ -399,32 +426,72 @@ export async function verifyWindowsAutoupdate( }); step('waiting for the upgraded app to relaunch automatically'); - const relaunchDeadline = Date.now() + 120_000; - let relaunched = []; - for (;;) { - relaunched = (await listInstalledProcesses(installDirectory)).filter( - (processInfo) => basename(processInfo.path).toLowerCase() === executableName.toLowerCase(), + const relaunched = await waitForInstalledProcessAppearance(installDirectory, executableName, { + timeoutMs: 120_000, + }); + const productVersion = await waitForInstalledProductVersion(installedExecutable, { run }); + try { + assertWindowsProductVersion(productVersion, nextVersion); + } catch (error) { + // Nondeterministic CI state observed once: a Maka.exe process is + // running from the install directory while the on-disk executable is + // still the previous version. The only code path in the NSIS template + // that launches the app is StartApp at full Section success, which + // contradicts old bytes on disk — so on mismatch, capture the raw + // state needed to attribute the launch and the transaction stage: + // each process's command line (StartApp passes `--updated`), the + // pre-upgrade backup directory (exists ⇒ the upgrade quit between + // customInit and customInstall), the uninstall registration, and the + // executable timestamps (rewritten ⇒ extraction ran). + const evidence = [`relaunched processes: ${JSON.stringify(relaunched)}`]; + try { + const commandLines = await run( + 'powershell', + [ + '-NoProfile', + '-NonInteractive', + '-Command', + `Get-CimInstance Win32_Process -Filter "Name='${executableName}'" | Select-Object ProcessId,ExecutablePath,CommandLine,CreationDate | ConvertTo-Json -Compress`, + ], + // Bounded: this probe runs on a machine already in a wedged state, + // and an evidence collector that hangs destroys the evidence — the + // catch below keeps a timeout from aborting the rest of the capture. + { timeoutMs: 30_000 }, + ); + evidence.push(`process command lines: ${commandLines.stdout.trim() || ''}`); + } catch (processError) { + evidence.push(`process command lines unavailable: ${processError.message}`); + } + const backupDirectory = `${installDirectory}.pre-upgrade-backup`; + evidence.push( + `backup directory ${backupDirectory}: ${existsSync(backupDirectory) ? 'present' : 'absent'}`, ); - if (relaunched.length > 0) break; - if (Date.now() >= relaunchDeadline) { - throw new Error('The installer did not relaunch the upgraded app within 120s.'); + try { + const registrations = await readUninstallDisplayVersions({ run }); + evidence.push(`uninstall registrations: ${JSON.stringify(registrations)}`); + } catch (registryError) { + evidence.push(`uninstall registrations unavailable: ${registryError.message}`); } - await delay(1_000); + for (const name of [executableName, uninstallExecutableName]) { + const filePath = join(installDirectory, name); + try { + const fileStat = await stat(filePath); + evidence.push(`${name}: mtime ${fileStat.mtime.toISOString()}, ${fileStat.size} bytes`); + } catch (statError) { + evidence.push(`${name}: ${statError.code ?? statError.message}`); + } + } + throw new Error(`${error.message}\n${evidence.join('\n')}`, { cause: error }); } - const productVersion = await readInstalledProductVersion(installedExecutable, { run }); - assertWindowsProductVersion(productVersion, nextVersion); step('stopping the relaunched instance'); // isForceRunAfter relaunches without our CDP/user-data arguments, so the // instance is observed (it exists, and the image on disk is the new // version) and then stopped before it can touch further state; the - // environment it inherited still points at the isolated home. - for (const processInfo of relaunched) { - await run('taskkill', ['/PID', String(processInfo.processId), '/T', '/F'], { - timeoutMs: 30_000, - }); - } - await waitForInstalledProcessesToExit(installDirectory); + // environment it inherited still points at the isolated home. The + // tolerant kill-then-prove policy lives in terminateInstalledProcesses, + // shared with the cleanup path below. + await terminateInstalledProcesses(installDirectory, { run }); step('running the full packaged smoke against the upgraded install'); // The sandbox probe writes its manifest into workingDirectory before the @@ -442,7 +509,6 @@ export async function verifyWindowsAutoupdate( requireWindowsSandbox: true, requireDisclaimer: true, smokeRenderer: async (executable, { workingDirectory }) => { - const smokePort = await reserveTcpPort(); const smokeHome = join(workingDirectory, 'home'); const smokeUserData = join(workingDirectory, 'user-data'); const smokeEnv = isolatedUserEnv(smokeHome); @@ -453,30 +519,36 @@ export async function verifyWindowsAutoupdate( const smokeChild = spawn( executable, [ - `--remote-debugging-port=${smokePort}`, + '--remote-debugging-port=0', `--user-data-dir=${smokeUserData}`, '--enable-logging=stderr', ], { cwd: workingDirectory, env: { ...process.env, MAKA_SKIP_SHELL_ENV: '1', ...smokeEnv }, - stdio: ['ignore', 'ignore', 'ignore'], + stdio: ['ignore', 'ignore', 'pipe'], }, ); + smokeChild.stderr.setEncoding('utf8'); + // A persistent collector, like every sibling smoke: without one the + // piped stream pauses once waitForDevToolsPort removes its listener, + // Chromium's stderr logging can fill the pipe and block the child, + // and the failure evidence this capture exists for is lost. + let smokeStderr = ''; + smokeChild.stderr.on('data', (chunk) => { + smokeStderr = `${smokeStderr}${chunk}`.slice(-16_384); + }); try { + const smokePort = await waitForDevToolsPort(smokeChild); const smokeTarget = await findRendererTarget(smokePort, smokeChild); - const deadline = Date.now() + 30_000; - for (;;) { - const state = await evaluateInRenderer( - smokeTarget.webSocketDebuggerUrl, - RENDERER_STATE_EXPRESSION, + await waitForUsableRenderer(smokeTarget.webSocketDebuggerUrl, smokeChild, { + description: 'Upgraded renderer', + }).catch((error) => { + throw new Error( + `${error.message}${smokeStderr.trim() ? `\n${smokeStderr.trim()}` : ''}`, + { cause: error }, ); - if (isPackagedRendererUsable(state)) break; - if (smokeChild.exitCode !== null || Date.now() >= deadline) { - throw new Error(`Upgraded renderer did not become usable: ${JSON.stringify(state)}`); - } - await delay(250); - } + }); const upgradedStatus = await evaluateInRenderer( smokeTarget.webSocketDebuggerUrl, upgradedStatusExpression, @@ -495,8 +567,12 @@ export async function verifyWindowsAutoupdate( await waitForInstalledProcessesToExit(installDirectory); step('uninstalling the upgraded application'); - await run(uninstaller, ['/S'], { timeoutMs: 120_000 }); - await waitUntilMissing(installDirectory); + // Files gone is not uninstall over: the detached uninstaller deletes the + // uninstall registry keys as its last action, tens of seconds later on a + // busy runner. Declaring success on files alone let the next verify step + // install inside that window and lose its fresh registration to this + // step's stale uninstaller (see waitForUninstallRegistrationToClear). + await completeInstalledApplicationUninstall(installDirectory, uninstaller, { run }); uninstallCompleted = true; step(`verified automatic update ${candidateVersion} -> ${nextVersion}`); return { candidateVersion, nextVersion, installDirectory }; @@ -522,21 +598,17 @@ export async function verifyWindowsAutoupdate( if (installationStarted && !uninstallCompleted) { let exited = false; try { - const leftover = await listInstalledProcesses(installDirectory); - for (const processInfo of leftover) { - await run('taskkill', ['/PID', String(processInfo.processId), '/T', '/F'], { - timeoutMs: 30_000, - }); - } - await waitForInstalledProcessesToExit(installDirectory); + // Same stop-and-prove policy as the main path: a stale-PID exit 128 + // from cleanup's own taskkill (run 32378497920) must not skip the + // authoritative exit wait and the uninstall/registration barrier. + await terminateInstalledProcesses(installDirectory, { run }); exited = true; } catch (error) { cleanupErrors.push(error); } if (exited) { try { - await access(uninstaller); - await run(uninstaller, ['/S'], { timeoutMs: 120_000 }); + await completeInstalledApplicationUninstall(installDirectory, uninstaller, { run }); } catch (error) { cleanupErrors.push(error); } diff --git a/scripts/verify-windows-harness.test.mjs b/scripts/verify-windows-harness.test.mjs new file mode 100644 index 0000000000..a0c714f56c --- /dev/null +++ b/scripts/verify-windows-harness.test.mjs @@ -0,0 +1,528 @@ +import assert from 'node:assert/strict'; +import { EventEmitter } from 'node:events'; +import { access, mkdir, mkdtemp, rm, symlink, writeFile } from 'node:fs/promises'; +import { createServer } from 'node:net'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { PassThrough } from 'node:stream'; +import { after, describe, it } from 'node:test'; +import { + diffTreeManifests, + directoryTreeManifest, + runCommand, + waitForDevToolsPort, + waitForUsableRenderer, +} from './verify-packaged-app.mjs'; +import { waitForInstalledProductVersion } from './verify-windows-autoupdate.mjs'; +import { + completeInstalledApplicationUninstall, + listInstalledProcesses, + terminateInstalledProcesses, + waitForInstalledProcessAppearance, + waitForInstalledProcessesToExit, + waitForUninstallRegistrationToClear, +} from './verify-windows-installer-lifecycle.mjs'; + +// Tests for the shared release-verification helpers. Everything here is +// platform-neutral on purpose: the Windows lanes execute these helpers for +// real, and this file proves their contracts on every PR that touches them. + +const temporaryRoots = []; +const delay = (milliseconds) => + new Promise((resolvePromise) => setTimeout(resolvePromise, milliseconds)); + +async function makeTree(shape) { + const root = await mkdtemp(join(tmpdir(), 'maka-harness-test-')); + temporaryRoots.push(root); + for (const [relative, content] of Object.entries(shape)) { + const absolute = join(root, relative); + if (content === null) { + await mkdir(absolute, { recursive: true }); + } else { + await mkdir(join(absolute, '..'), { recursive: true }); + await writeFile(absolute, content); + } + } + return root; +} + +after(async () => { + for (const root of temporaryRoots) { + await rm(root, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + } +}); + +describe('directoryTreeManifest', () => { + it('returns sorted POSIX-relative paths with sha256 content hashes', async () => { + const root = await makeTree({ + 'b.txt': 'bee', + 'a/nested.txt': 'nested', + 'a/z.txt': 'zed', + }); + const manifest = await directoryTreeManifest(root); + assert.deepEqual( + manifest.map((entry) => entry.path), + ['a/nested.txt', 'a/z.txt', 'b.txt'], + ); + for (const entry of manifest) { + assert.match(entry.sha256, /^[0-9a-f]{64}$/); + } + }); + + it('records empty directories so their loss is visible', async () => { + const root = await makeTree({ 'a/file.txt': 'x', 'a/empty': null }); + const manifest = await directoryTreeManifest(root); + assert.deepEqual( + manifest.map((entry) => entry.path), + ['a/empty/', 'a/file.txt'], + ); + assert.equal(manifest.find((entry) => entry.path === 'a/empty/').sha256, null); + }); + + it('hashes content, not names: same names different bytes differ', async () => { + const left = await directoryTreeManifest(await makeTree({ 'f.txt': 'one' })); + const right = await directoryTreeManifest(await makeTree({ 'f.txt': 'two' })); + assert.notEqual(left[0].sha256, right[0].sha256); + }); + + it('throws on non-regular entries instead of skipping them', async (t) => { + const root = await makeTree({ 'real.txt': 'x' }); + try { + await symlink(join(root, 'real.txt'), join(root, 'link.txt')); + } catch (error) { + // Windows without Developer Mode cannot create symlinks; the guard + // itself is platform-neutral readdir dirent logic. + if (error.code === 'EPERM') return t.skip('symlink creation requires privilege here'); + throw error; + } + await assert.rejects( + () => directoryTreeManifest(root), + /Unsupported directory entry .*link\.txt/, + ); + }); +}); + +describe('diffTreeManifests', () => { + const manifest = (entries) => entries.map(([path, sha256]) => ({ path, sha256 })); + const cases = [ + { + name: 'identical manifests diff empty', + before: manifest([ + ['a.txt', '1'.repeat(64)], + ['b/c.txt', '2'.repeat(64)], + ]), + after: manifest([ + ['a.txt', '1'.repeat(64)], + ['b/c.txt', '2'.repeat(64)], + ]), + expected: { missing: [], extra: [], changed: [] }, + }, + { + name: 'a lost file is missing', + before: manifest([['a.txt', '1'.repeat(64)]]), + after: manifest([]), + expected: { missing: ['a.txt'], extra: [], changed: [] }, + }, + { + name: 'a new file is extra', + before: manifest([]), + after: manifest([['n.txt', '3'.repeat(64)]]), + expected: { missing: [], extra: ['n.txt'], changed: [] }, + }, + { + name: 'different bytes at the same path are changed', + before: manifest([['a.txt', '1'.repeat(64)]]), + after: manifest([['a.txt', '4'.repeat(64)]]), + expected: { missing: [], extra: [], changed: ['a.txt'] }, + }, + { + name: 'a lost empty directory is missing', + before: manifest([['a/empty/', null]]), + after: manifest([]), + expected: { missing: ['a/empty/'], extra: [], changed: [] }, + }, + ]; + for (const { name, before, after: afterManifest, expected } of cases) { + it(name, () => { + assert.deepEqual(diffTreeManifests(before, afterManifest), expected); + }); + } +}); + +describe('runCommand', () => { + it('kills the child and rejects when timeoutMs elapses', async () => { + const root = await makeTree({}); + const sentinel = join(root, 'child-survived.txt'); + await assert.rejects( + () => + runCommand( + process.execPath, + [ + '-e', + "setTimeout(() => require('node:fs').writeFileSync(process.argv[1], 'leaked'), 600)", + sentinel, + ], + { timeoutMs: 100 }, + ), + /did not finish within 100ms/, + ); + await delay(800); + await assert.rejects(() => access(sentinel), { code: 'ENOENT' }); + }); + + it('resolves stdout for a completing command without a timeout', async () => { + const { stdout } = await runCommand(process.execPath, ['-e', "process.stdout.write('ok')"]); + assert.equal(stdout, 'ok'); + }); +}); + +describe('waitForDevToolsPort', () => { + const makeChild = () => { + const child = new EventEmitter(); + child.stderr = new PassThrough(); + child.exitCode = null; + return child; + }; + + it('resolves the announced port', async () => { + const child = makeChild(); + const wait = waitForDevToolsPort(child, { timeoutMs: 2_000 }); + child.stderr.write('DevTools listening on ws://127.0.0.1:54321/devtools/browser/abc\n'); + assert.equal(await wait, 54321); + }); + + it('rejects when the child exits before announcing', async () => { + const child = makeChild(); + const wait = waitForDevToolsPort(child, { timeoutMs: 2_000 }); + child.exitCode = 1; + child.emit('exit'); + await assert.rejects(() => wait, /exited before announcing/); + }); +}); + +describe('waitForUsableRenderer', () => { + it('fails fast when the child is already dead', async () => { + await assert.rejects( + () => + waitForUsableRenderer( + 'ws://127.0.0.1:1/devtools/page/x', + { exitCode: 1 }, + { + deadlineMs: 5_000, + description: 'T', + }, + ), + /T exited before it became usable/, + ); + }); + + it('retries failed probes and fails only at the deadline', async () => { + const startedAt = Date.now(); + await assert.rejects( + () => + waitForUsableRenderer( + 'ws://127.0.0.1:1/devtools/page/x', + { exitCode: null }, + { + deadlineMs: 700, + description: 'T', + }, + ), + /T did not become usable within 700ms/, + ); + const elapsed = Date.now() - startedAt; + assert.ok(elapsed >= 700, `deadline respected: ${elapsed}`); + assert.ok(elapsed < 30_000, `no runaway: ${elapsed}`); + }); + + it('caps a half-open WebSocket probe to the remaining deadline budget', async () => { + const sockets = new Set(); + const server = createServer((socket) => { + sockets.add(socket); + socket.on('close', () => sockets.delete(socket)); + }); + await new Promise((resolvePromise) => server.listen(0, '127.0.0.1', resolvePromise)); + const address = server.address(); + assert.ok(address && typeof address !== 'string'); + const startedAt = Date.now(); + try { + await assert.rejects( + () => + waitForUsableRenderer( + `ws://127.0.0.1:${address.port}/devtools/page/hung`, + { exitCode: null }, + { deadlineMs: 200, description: 'T' }, + ), + /T did not become usable within 200ms/, + ); + const elapsed = Date.now() - startedAt; + assert.ok(elapsed < 2_000, `inner probe exceeded the outer deadline budget: ${elapsed}`); + } finally { + for (const socket of sockets) socket.destroy(); + await new Promise((resolvePromise) => server.close(resolvePromise)); + } + }); +}); + +describe('waitForInstalledProcessesToExit', () => { + it('returns once an enumeration reports no processes', async () => { + const snapshots = [[{ processId: 1, name: 'Maka.exe' }], []]; + await waitForInstalledProcessesToExit('C:/nowhere', { + listProcesses: async () => snapshots.shift(), + timeoutMs: 2_000, + pollIntervalMs: 10, + }); + }); + + it('never treats a failed enumeration as empty: rejects at the deadline', async () => { + await assert.rejects( + () => + waitForInstalledProcessesToExit('C:/nowhere', { + listProcesses: async () => { + throw new Error('wmi stalled'); + }, + timeoutMs: 200, + pollIntervalMs: 10, + }), + /Could not enumerate installed Maka processes within 200ms: wmi stalled/, + ); + }); +}); + +describe('listInstalledProcesses', () => { + it('bounds a WMI round to ten seconds so polling can retry', async () => { + let observedTimeout; + const processes = await listInstalledProcesses('C:/nowhere', { + run: async (_command, _args, options) => { + observedTimeout = options.timeoutMs; + return { stdout: '', stderr: '' }; + }, + }); + assert.deepEqual(processes, []); + assert.equal(observedTimeout, 10_000); + }); +}); + +describe('waitForInstalledProcessAppearance', () => { + it('survives a failed probe and returns the later successful match', async () => { + // The regression run 32340493254 rejected on the first wedged + // enumeration; the contract is failure-then-success within the deadline. + const probes = [ + () => Promise.reject(new Error('wmi stalled')), + () => + Promise.resolve([ + { processId: 7, name: 'Maka.exe', path: 'C:/nowhere/installed/Maka.exe' }, + ]), + ]; + const matched = await waitForInstalledProcessAppearance('C:/nowhere/installed', 'Maka.exe', { + listProcesses: () => probes.shift()(), + timeoutMs: 2_000, + pollIntervalMs: 10, + }); + assert.deepEqual( + matched.map((processInfo) => processInfo.processId), + [7], + ); + }); + + it('matches by executable basename, case-insensitively', async () => { + const matched = await waitForInstalledProcessAppearance('C:/nowhere/installed', 'Maka.exe', { + listProcesses: async () => [ + { processId: 1, name: 'other.exe', path: 'C:/nowhere/installed/other.exe' }, + { processId: 2, name: 'MAKA.EXE', path: 'C:/nowhere/installed/MAKA.EXE' }, + ], + timeoutMs: 2_000, + pollIntervalMs: 10, + }); + assert.deepEqual( + matched.map((processInfo) => processInfo.processId), + [2], + ); + }); + + it('reports the last probe error only when no later enumeration succeeds', async () => { + await assert.rejects( + () => + waitForInstalledProcessAppearance('C:/nowhere/installed', 'Maka.exe', { + listProcesses: async () => { + throw new Error('wmi stalled'); + }, + timeoutMs: 150, + pollIntervalMs: 10, + }), + /did not appear among installed processes within 150ms\.\nlast probe error: wmi stalled/, + ); + }); +}); + +describe('waitForInstalledProductVersion', () => { + it('retries a stalled PowerShell read within the outer deadline', async () => { + const observedTimeouts = []; + const results = [new Error('powershell stalled'), { stdout: '0.1.12', stderr: '' }]; + const version = await waitForInstalledProductVersion('C:/installed/Maka.exe', { + run: async (_command, _args, options) => { + observedTimeouts.push(options.timeoutMs); + const result = results.shift(); + if (result instanceof Error) throw result; + return result; + }, + timeoutMs: 60_000, + pollIntervalMs: 1, + }); + assert.equal(version, '0.1.12'); + assert.equal(observedTimeouts.length, 2); + assert.ok(observedTimeouts.every((timeout) => timeout > 0 && timeout <= 10_000)); + }); +}); + +describe('terminateInstalledProcesses', () => { + const process7 = { processId: 7, name: 'Maka.exe', path: 'C:/nowhere/installed/Maka.exe' }; + + it('tolerates exit 128 and an overrun kill, then proves exit', async () => { + const events = []; + const failures = [ + new Error('taskkill /PID 7 /T /F failed with exit code 128'), + new Error('taskkill /PID 8 /T /F did not finish within 30000ms'), + ]; + await terminateInstalledProcesses('C:/nowhere/installed', { + listProcesses: async () => [process7, { ...process7, processId: 8 }], + run: async (command, args) => { + events.push(`${command}:${args[1]}`); + throw failures.shift(); + }, + waitForExit: async () => { + events.push('wait'); + }, + }); + // Both mechanism failures were tolerated and the authoritative exit + // wait still ran. + assert.deepEqual(events, ['taskkill:7', 'taskkill:8', 'wait']); + }); + + it('lets the exit proof decide after any taskkill mechanism failure', async () => { + let exitProofRan = false; + await terminateInstalledProcesses('C:/nowhere/installed', { + listProcesses: async () => [process7], + run: async () => { + throw new Error('taskkill /PID 7 /T /F failed with exit code 255'); + }, + waitForExit: async () => { + exitProofRan = true; + }, + }); + assert.equal(exitProofRan, true); + }); + + it('reports the exit proof and taskkill failures together when residue remains', async () => { + await assert.rejects( + () => + terminateInstalledProcesses('C:/nowhere/installed', { + listProcesses: async () => [process7], + run: async () => { + throw new Error('taskkill failed with exit code 255'); + }, + waitForExit: async () => { + throw new Error('Maka.exe (7) is still running'); + }, + }), + (error) => { + assert.ok(error instanceof AggregateError); + assert.match(error.message, /did not exit after taskkill failures/); + assert.deepEqual( + error.errors.map((failure) => failure.message), + ['Maka.exe (7) is still running', 'taskkill failed with exit code 255'], + ); + return true; + }, + ); + }); + + it('retries a transient enumeration failure before killing and proving exit', async () => { + const events = []; + let attempts = 0; + await terminateInstalledProcesses('C:/nowhere/installed', { + listProcesses: async () => { + attempts += 1; + if (attempts === 1) throw new Error('wmi stalled'); + return [process7]; + }, + run: async () => { + events.push('kill'); + return { stdout: '', stderr: '' }; + }, + waitForExit: async () => events.push('wait'), + pollIntervalMs: 1, + }); + assert.equal(attempts, 2); + assert.deepEqual(events, ['kill', 'wait']); + }); +}); + +describe('completeInstalledApplicationUninstall', () => { + it('runs the uninstaller and waits for both cleanup barriers in order', async () => { + const events = []; + await completeInstalledApplicationUninstall('C:/installed', 'C:/installed/uninstall.exe', { + requirePath: async () => events.push('access'), + run: async () => { + events.push('uninstall'); + return { stdout: '', stderr: '' }; + }, + waitForMissing: async () => events.push('files-missing'), + waitForRegistration: async () => events.push('registration-clear'), + }); + assert.deepEqual(events, ['access', 'uninstall', 'files-missing', 'registration-clear']); + }); + + it('still proves both barriers when a detached uninstaller is already gone', async () => { + const events = []; + await completeInstalledApplicationUninstall('C:/installed', 'C:/installed/uninstall.exe', { + requirePath: async () => { + const error = new Error('missing'); + error.code = 'ENOENT'; + throw error; + }, + run: async () => assert.fail('a missing uninstaller must not be launched'), + waitForMissing: async () => events.push('files-missing'), + waitForRegistration: async () => events.push('registration-clear'), + }); + assert.deepEqual(events, ['files-missing', 'registration-clear']); + }); +}); + +describe('waitForUninstallRegistrationToClear', () => { + const runReturning = (outputs) => async () => ({ stdout: outputs.shift(), stderr: '' }); + + it('returns once the registration reads empty', async () => { + await waitForUninstallRegistrationToClear({ + run: runReturning(['0.1.11\n', '\n']), + timeoutMs: 2_000, + pollIntervalMs: 10, + }); + }); + + it('reports the lingering registration at the deadline', async () => { + await assert.rejects( + () => + waitForUninstallRegistrationToClear({ + run: async () => ({ stdout: '0.1.11\n', stderr: '' }), + timeoutMs: 150, + pollIntervalMs: 10, + }), + /uninstall registration \(DisplayVersion "0\.1\.11"\) is still present after 150ms/, + ); + }); + + it('tolerates probe failures but rejects with them at the deadline', async () => { + await assert.rejects( + () => + waitForUninstallRegistrationToClear({ + run: async () => { + throw new Error('registry stalled'); + }, + timeoutMs: 150, + pollIntervalMs: 10, + }), + /Could not read the Maka uninstall registration within 150ms: registry stalled/, + ); + }); +}); diff --git a/scripts/verify-windows-installer-lifecycle.mjs b/scripts/verify-windows-installer-lifecycle.mjs index 4f20445308..e229ad332d 100644 --- a/scripts/verify-windows-installer-lifecycle.mjs +++ b/scripts/verify-windows-installer-lifecycle.mjs @@ -8,6 +8,7 @@ import { powerShellLiteral, verifyPackagedWindowsApp } from './verify-windows-x6 const uninstallExecutableName = 'Uninstall Maka.exe'; const temporaryCleanupRetries = 20; const temporaryCleanupRetryDelayMs = 250; +const pollingProbeTimeoutMs = 10_000; export function installerVersion(path) { const match = basename(path).match(/^Maka-(\d+\.\d+\.\d+)-win-x64\.exe$/u); @@ -21,7 +22,10 @@ function delay(milliseconds) { return new Promise((resolvePromise) => setTimeout(resolvePromise, milliseconds)); } -export async function listInstalledProcesses(installDirectory, { run = runCommand } = {}) { +export async function listInstalledProcesses( + installDirectory, + { run = runCommand, timeoutMs = pollingProbeTimeoutMs } = {}, +) { const root = `${resolve(installDirectory)}${sep}`; const script = String.raw` $root = [IO.Path]::GetFullPath(${powerShellLiteral(root)}) @@ -37,7 +41,16 @@ $matches = @( ) $matches | ConvertTo-Json -Compress `; - const { stdout } = await run('powershell', ['-NoProfile', '-NonInteractive', '-Command', script]); + // Bounded: this probe runs inside polling loops whose deadline is the + // authority. An unbounded WMI query that wedges would otherwise turn a + // 60-second wait into the workflow timeout with no diagnostic. + const { stdout } = await run( + 'powershell', + ['-NoProfile', '-NonInteractive', '-Command', script], + { + timeoutMs, + }, + ); if (!stdout.trim()) return []; const parsed = JSON.parse(stdout); return Array.isArray(parsed) ? parsed : [parsed]; @@ -53,16 +66,144 @@ export async function waitForInstalledProcessesToExit( } = {}, ) { const deadline = Date.now() + timeoutMs; - let processes = await listProcesses(installDirectory); - while (processes.length > 0) { + let lastProbeError; + for (;;) { + // A failed enumeration is retried, never treated as "no processes": + // this wait exists to prove exit, so an unreadable state must keep + // polling and fail at the deadline, not pass or die on one hiccup. + let processes; + try { + processes = await listProcesses(installDirectory, { + timeoutMs: remainingProbeBudget(deadline), + }); + lastProbeError = undefined; + } catch (error) { + lastProbeError = error; + } + if (processes && processes.length === 0) return; if (Date.now() >= deadline) { + if (lastProbeError) { + throw new Error( + `Could not enumerate installed Maka processes within ${timeoutMs}ms: ` + + `${lastProbeError.message}`, + { cause: lastProbeError }, + ); + } const summary = processes.map(({ processId, name }) => `${name} (${processId})`).join(', '); throw new Error( `Installed Maka processes did not exit within ${timeoutMs}ms: ${summary || ''}.`, ); } await sleep(pollIntervalMs); - processes = await listProcesses(installDirectory); + } +} + +/** + * Wait for a named executable to appear among the installation's processes. + * The same probe-tolerance contract as the exit wait above, in the opposite + * direction: one transient WMI failure is one bad probe, not a verdict (run + * 32340493254 failed the relaunch wait on a single wedged enumeration while + * the upgraded app may already have been running). The deadline is the + * authority; the last probe error is evidence only when no later enumeration + * succeeds. + */ +export async function waitForInstalledProcessAppearance( + installDirectory, + executableName, + { + listProcesses = listInstalledProcesses, + timeoutMs = 120_000, + pollIntervalMs = 1_000, + sleep = delay, + } = {}, +) { + const deadline = Date.now() + timeoutMs; + let lastProbeError; + for (;;) { + try { + const matched = ( + await listProcesses(installDirectory, { + timeoutMs: remainingProbeBudget(deadline), + }) + ).filter( + (processInfo) => basename(processInfo.path).toLowerCase() === executableName.toLowerCase(), + ); + lastProbeError = undefined; + if (matched.length > 0) return matched; + } catch (error) { + lastProbeError = error; + } + if (Date.now() >= deadline) { + throw new Error( + `${executableName} did not appear among installed processes within ${timeoutMs}ms.` + + `${lastProbeError ? `\nlast probe error: ${lastProbeError.message}` : ''}`, + lastProbeError ? { cause: lastProbeError } : undefined, + ); + } + await sleep(pollIntervalMs); + } +} + +/** + * Stop every process running from the installation directory and prove they + * are gone. The kill is the mechanism, not the assertion: exit 128 (the tree + * was already gone) and a taskkill that overruns its own bound (observed on + * wedged runners, run 32378497920, while the target still died) are + * tolerated, because the authoritative check is the exit wait, which fails + * with the live process list if anything from the install tree still runs. + * One policy for the main path and cleanup — run 32378497920's cleanup hit + * the same stale-PID exit 128 through a second, stricter copy of this loop. + */ +export async function terminateInstalledProcesses( + installDirectory, + { + run = runCommand, + listProcesses = listInstalledProcesses, + waitForExit = waitForInstalledProcessesToExit, + enumerationTimeoutMs = 60_000, + pollIntervalMs = 1_000, + sleep = delay, + } = {}, +) { + const deadline = Date.now() + enumerationTimeoutMs; + const enumerate = (directory, options = {}) => listProcesses(directory, { run, ...options }); + let processes; + for (;;) { + try { + processes = await enumerate(installDirectory, { + timeoutMs: remainingProbeBudget(deadline), + }); + break; + } catch (error) { + if (Date.now() >= deadline) { + throw new Error( + `Could not enumerate installed Maka processes within ${enumerationTimeoutMs}ms: ` + + `${error.message}`, + { cause: error }, + ); + } + await sleep(pollIntervalMs); + } + } + const killErrors = []; + for (const processInfo of processes) { + try { + await run('taskkill', ['/PID', String(processInfo.processId), '/T', '/F'], { + timeoutMs: 30_000, + }); + } catch (error) { + killErrors.push(error); + } + } + try { + await waitForExit(installDirectory, { listProcesses: enumerate }); + } catch (exitError) { + if (killErrors.length === 0) throw exitError; + throw new AggregateError( + [exitError, ...killErrors], + 'Installed Maka processes did not exit after taskkill failures.', + { cause: exitError }, + ); } } @@ -85,6 +226,125 @@ export async function waitUntilMissing( } } +/** + * Every HKCU `DisplayVersion` registered under the Maka display name, joined + * with commas ('' when none). One PowerShell scan, shared by the harnesses + * that assert on — or wait out — the uninstall registration. + */ +export async function readUninstallDisplayVersions({ + run = runCommand, + timeoutMs = pollingProbeTimeoutMs, +} = {}) { + // electron-builder's default uninstallDisplayName is + // "${productName} ${version}" (NsisTarget: UNINSTALL_DISPLAY_NAME), so the + // registered DisplayName is "Maka 0.1.11", never the bare product name. The + // trailing space in the wildcard keeps other products starting with "Maka" + // out; the bare-name arm covers a configured uninstallDisplayName without a + // version, should one ever be set. + const script = String.raw` +$entries = Get-ChildItem 'HKCU:\Software\Microsoft\Windows\CurrentVersion\Uninstall' | + ForEach-Object { Get-ItemProperty $_.PSPath } | + Where-Object { $_.DisplayName -eq 'Maka' -or $_.DisplayName -like 'Maka *' } +@($entries | ForEach-Object { $_.DisplayVersion }) -join ',' +`; + // Bounded for the same reason as listInstalledProcesses: the enclosing + // waits own the deadline, so one stalled registry query must fail this + // probe, not the whole lane. + const { stdout } = await run( + 'powershell', + ['-NoProfile', '-NonInteractive', '-Command', script], + { + timeoutMs, + }, + ); + return stdout.trim(); +} + +/** + * An NSIS uninstall is not over when the files are gone. Launched without + * `_?=`, "Uninstall Maka.exe" copies itself to %TEMP% and detaches; the copy + * removes $INSTDIR first and deletes the uninstall/install registry keys as + * its LAST action (uninstaller.nsh: RMDir /r, then shortcuts, SHChangeNotify + * and app-data handling, then DeleteRegKey), which on a busy runner lands tens + * of seconds after `waitUntilMissing` saw the files disappear. Anything that + * installs into that window gets its freshly written registration deleted by + * the stale uninstaller — observed as an empty uninstall entry two steps + * later. The uninstall registration's disappearance is the near-final + * signal, not the final one: the uninstaller deletes the uninstall key at + * uninstaller.nsh:250 and the install key at :254, so when the registration + * vanishes the detached copy still has one registry deletion left. A fresh + * install writes both keys after this wait returns, so the residual window + * is the width of two adjacent registry calls — accepted, and stated here + * so the premise is re-checked if the uninstaller's ordering ever changes. + */ +export async function waitForUninstallRegistrationToClear({ + run = runCommand, + timeoutMs = 120_000, + pollIntervalMs = 2_000, +} = {}) { + const deadline = Date.now() + timeoutMs; + let lastProbeError; + for (;;) { + // One stalled or failed probe is tolerated and retried — the deadline + // below is the authority, and a transient WMI/registry hiccup must not + // end a wait that owns two minutes of budget. + let versions; + try { + versions = await readUninstallDisplayVersions({ + run, + timeoutMs: remainingProbeBudget(deadline), + }); + lastProbeError = undefined; + } catch (error) { + lastProbeError = error; + } + if (versions === '') return; + if (Date.now() >= deadline) { + if (lastProbeError) { + throw new Error( + `Could not read the Maka uninstall registration within ${timeoutMs}ms: ` + + `${lastProbeError.message}`, + { cause: lastProbeError }, + ); + } + throw new Error( + `A Maka uninstall registration (DisplayVersion ${JSON.stringify(versions)}) is still ` + + `present after ${timeoutMs}ms — a detached uninstaller has not finished, or an ` + + `earlier step leaked an installation.`, + ); + } + await delay(pollIntervalMs); + } +} + +export async function completeInstalledApplicationUninstall( + installDirectory, + uninstaller, + { + run = runCommand, + requirePath = access, + waitForMissing = waitUntilMissing, + waitForRegistration = waitForUninstallRegistrationToClear, + } = {}, +) { + let uninstallerExists = true; + try { + await requirePath(uninstaller); + } catch (error) { + if (error?.code !== 'ENOENT') throw error; + uninstallerExists = false; + } + if (uninstallerExists) { + await run(uninstaller, ['/S'], { timeoutMs: 120_000 }); + } + await waitForMissing(installDirectory); + await waitForRegistration({ run }); +} + +function remainingProbeBudget(deadline) { + return Math.max(1, Math.min(pollingProbeTimeoutMs, deadline - Date.now())); +} + export async function verifyWindowsInstallerLifecycle( inputPath, previousInputPath, @@ -158,8 +418,14 @@ export async function verifyWindowsInstallerLifecycle( await waitForProcessesToExit(installDirectory); console.log('[verify-windows-installer] uninstalling'); - await run(uninstaller, ['/S'], { timeoutMs: 120_000 }); - await waitForMissing(installDirectory); + // The registration's disappearance, not the files', is the uninstall's + // final action; the next verify step in this job installs immediately and + // must not race the detached uninstaller's DeleteRegKey. + await completeInstalledApplicationUninstall(installDirectory, uninstaller, { + run, + requirePath, + waitForMissing, + }); uninstallCompleted = true; console.log('[verify-windows-installer] lifecycle verified'); @@ -179,8 +445,11 @@ export async function verifyWindowsInstallerLifecycle( } if (installedProcessesExited) { try { - await requirePath(uninstaller); - await run(uninstaller, ['/S'], { timeoutMs: 120_000 }); + await completeInstalledApplicationUninstall(installDirectory, uninstaller, { + run, + requirePath, + waitForMissing, + }); } catch (error) { cleanupErrors.push(error); }