From 45eaccab4b911a27f6145b733977db40f2e7d413 Mon Sep 17 00:00:00 2001 From: Dongdong Chen Date: Thu, 19 Nov 2026 02:53:02 +0800 Subject: [PATCH 1/3] fix(desktop): update localmodels to 0.1.0.dev40 0.1.0.dev40 replaces a Pi that cannot load extensions when the stack is applied again, reports it in setup binding as pi-extensions-unavailable, fails harness apply with RUNTIME_ACTION_FAILED or RUNTIME_EXTENSIONS_FAILED instead of exiting 0 after an incomplete install, refuses an unusable SIMPLEX_PI_PATH and a too-long Windows runtime root at plan time, reports GPU and free-memory shortfalls with byte figures, and stops the library Bifrost on stack down. - Pin 0.1.0.dev40 in the three desktop distributions and in scripts/installs/stack.sh and stack.ps1, copied from the published launchers; the three tool archives and both launchers were checked against the release's SHA256 and size. - Point the run-without-cloning URLs in scripts/installs/README.md at the pinned version, and sync the stack pins in scripts/install with scripts/sync-install-stack-pin.mjs. Co-Authored-By: Claude Opus 5.5 --- apps/desktop/distribution/darwin-arm64.json | 10 ++-- apps/desktop/distribution/linux-x64.json | 10 ++-- apps/desktop/distribution/win32-x64.json | 10 ++-- scripts/install/install.ps1 | 4 +- scripts/install/install.sh | 4 +- scripts/installs/README.md | 8 +-- scripts/installs/stack.ps1 | 66 ++++++++++----------- scripts/installs/stack.sh | 22 +++---- 8 files changed, 67 insertions(+), 67 deletions(-) diff --git a/apps/desktop/distribution/darwin-arm64.json b/apps/desktop/distribution/darwin-arm64.json index d6c637ad..391fb54b 100644 --- a/apps/desktop/distribution/darwin-arm64.json +++ b/apps/desktop/distribution/darwin-arm64.json @@ -2,11 +2,11 @@ "platform": "darwin", "arch": "arm64", "localmodelsTool": { - "url": "https://d2ikbbcl3j6jz9.cloudfront.net/cli/localmodels/0.1.0.dev39/darwin-arm64.tgz", - "sha256": "e15c1d96c4db70cc834b42693228c6b6bbe7eb575e79d19e8aff2f0bf8ba6e48", - "bytes": 14937177, - "sourceCommit": "9a72159a9c114e323de703fcbf6e030fbf7d480a", - "manifestSha256": "b9f750c55834913394ebc459395f5331915d02d98ebeecee3586fb1721a7bc41" + "url": "https://d2ikbbcl3j6jz9.cloudfront.net/cli/localmodels/0.1.0.dev40/darwin-arm64.tgz", + "sha256": "648e8909915d8da2b87ae4f4797cd22f2d46d8a7cd025287ab9e6b5436119687", + "bytes": 14953559, + "sourceCommit": "a4de3e359d2a4f9b654f9ae7bdf6887c55896b7d", + "manifestSha256": "15f7138847ef4d13f9c2df455ec04423dc35d30d759d5f952dcb0b69d81db13b" }, "nativeSources": [ { diff --git a/apps/desktop/distribution/linux-x64.json b/apps/desktop/distribution/linux-x64.json index a425f170..ad7b3ee7 100644 --- a/apps/desktop/distribution/linux-x64.json +++ b/apps/desktop/distribution/linux-x64.json @@ -2,11 +2,11 @@ "platform": "linux", "arch": "x64", "localmodelsTool": { - "url": "https://d2ikbbcl3j6jz9.cloudfront.net/cli/localmodels/0.1.0.dev39/linux-x64.tgz", - "sha256": "f58f55f94e3559ffd959af9e9f44663a41ec34aef53a1c2d1ff7279ec207af3a", - "bytes": 20776267, - "sourceCommit": "9a72159a9c114e323de703fcbf6e030fbf7d480a", - "manifestSha256": "60664ab53e44b9967bf21e54a2e2fb263862296537ebd24691ab96e7ed268336" + "url": "https://d2ikbbcl3j6jz9.cloudfront.net/cli/localmodels/0.1.0.dev40/linux-x64.tgz", + "sha256": "a8e176fa426360f489bf9c552064faf7e1dd88072b463a13185ff1193a223683", + "bytes": 20790291, + "sourceCommit": "a4de3e359d2a4f9b654f9ae7bdf6887c55896b7d", + "manifestSha256": "d36a909ddad7be0de0642766102a5478efd22d7d437c86d8b63ac7bddc24e082" }, "nativeSources": [ { diff --git a/apps/desktop/distribution/win32-x64.json b/apps/desktop/distribution/win32-x64.json index f20abd5f..26966b59 100644 --- a/apps/desktop/distribution/win32-x64.json +++ b/apps/desktop/distribution/win32-x64.json @@ -2,11 +2,11 @@ "platform": "win32", "arch": "x64", "localmodelsTool": { - "url": "https://d2ikbbcl3j6jz9.cloudfront.net/cli/localmodels/0.1.0.dev39/win32-x64.tgz", - "sha256": "492ed40845a7d732f077b575c7ef4d75f06965c991e5e2c4c0e9f356ed0da37f", - "bytes": 23174114, - "sourceCommit": "9a72159a9c114e323de703fcbf6e030fbf7d480a", - "manifestSha256": "f17122b311b71a3b37e3207a2c0cdce6f84b3a14e09ef0bceed81be74e90e161" + "url": "https://d2ikbbcl3j6jz9.cloudfront.net/cli/localmodels/0.1.0.dev40/win32-x64.tgz", + "sha256": "de07306075b5efbf2158c41a7c87fe4939bac45b9b6f953270ed00017a6540be", + "bytes": 23191788, + "sourceCommit": "a4de3e359d2a4f9b654f9ae7bdf6887c55896b7d", + "manifestSha256": "c7e2fab80d7aa2bdf52975a0b6e180611830a5d3c013bb302d16ff5c3c0d42dd" }, "nativeSources": [ { diff --git a/scripts/install/install.ps1 b/scripts/install/install.ps1 index 0d46b2e9..98a48d67 100644 --- a/scripts/install/install.ps1 +++ b/scripts/install/install.ps1 @@ -167,8 +167,8 @@ $script:PowerShellExe = (Get-Process -Id $PID).Path $script:Utf8NoBom = New-Object System.Text.UTF8Encoding($false) $script:TempDirsToClean = New-Object System.Collections.Generic.List[string] # >>> stack release pin -$script:StackPinNpmVersion = '0.1.0-dev.39' -$script:StackPinSha256 = '089102538c287af8bc156a5b3c9b90815b39392b86e1ac0deb2bae2a6f9ff80c' +$script:StackPinNpmVersion = '0.1.0-dev.40' +$script:StackPinSha256 = '04fff205b3ac826caacb8df2e20c7f11f4a10a0fbdd81e4d254d9e3d136b503a' # <<< stack release pin $script:StackScriptPath = '' $script:KitHome = '' diff --git a/scripts/install/install.sh b/scripts/install/install.sh index f1e15011..270f08b3 100755 --- a/scripts/install/install.sh +++ b/scripts/install/install.sh @@ -42,8 +42,8 @@ KIT_PYTHON="" PROMPT_VALUE="" # >>> stack release pin -STACK_PIN_NPM_VERSION='0.1.0-dev.39' -STACK_PIN_SHA256='49280238b7e92972addad17436f81890ce4d19f78d3d0652b73c1e1c745a3e55' +STACK_PIN_NPM_VERSION='0.1.0-dev.40' +STACK_PIN_SHA256='f4bc079520e73fdde9f15fe050397a46cde27fcc1941702b70dd8c287b1b57ee' # <<< stack release pin usage() { diff --git a/scripts/installs/README.md b/scripts/installs/README.md index a4a0f660..496315ae 100644 --- a/scripts/installs/README.md +++ b/scripts/installs/README.md @@ -89,15 +89,15 @@ Run the pinned script without cloning the repository: ```sh # global -curl -fsSL https://cdn.jsdelivr.net/npm/@simplexai/localmodels@0.1.0-dev.39/stack.sh | sh -s -- --region global --help +curl -fsSL https://cdn.jsdelivr.net/npm/@simplexai/localmodels@0.1.0-dev.40/stack.sh | sh -s -- --region global --help # cn -curl -fsSL https://registry.npmmirror.com/@simplexai/localmodels/0.1.0-dev.39/files/stack.sh | sh -s -- --region cn --help +curl -fsSL https://registry.npmmirror.com/@simplexai/localmodels/0.1.0-dev.40/files/stack.sh | sh -s -- --region cn --help ``` ```powershell -irm https://cdn.jsdelivr.net/npm/@simplexai/localmodels@0.1.0-dev.39/stack.ps1 -OutFile stack.ps1 +irm https://cdn.jsdelivr.net/npm/@simplexai/localmodels@0.1.0-dev.40/stack.ps1 -OutFile stack.ps1 powershell -ExecutionPolicy Bypass -File .\stack.ps1 -Region global --help -# cn script URL: https://registry.npmmirror.com/@simplexai/localmodels/0.1.0-dev.39/files/stack.ps1 +# cn script URL: https://registry.npmmirror.com/@simplexai/localmodels/0.1.0-dev.40/files/stack.ps1 ``` The localmodels release process publishes each version's platform kits, its patched Bifrost packages and the diff --git a/scripts/installs/stack.ps1 b/scripts/installs/stack.ps1 index 3492c67e..2d7422e3 100644 --- a/scripts/installs/stack.ps1 +++ b/scripts/installs/stack.ps1 @@ -3,59 +3,59 @@ $ErrorActionPreference = 'Stop' $ProgressPreference = 'SilentlyContinue' # >>> release pin -$StackVersion = '0.1.0.dev39' -$StackSourceCommit = '9a72159a9c114e323de703fcbf6e030fbf7d480a' -$StackNpmVersion = '0.1.0-dev.39' +$StackVersion = '0.1.0.dev40' +$StackSourceCommit = 'a4de3e359d2a4f9b654f9ae7bdf6887c55896b7d' +$StackNpmVersion = '0.1.0-dev.40' $StackKits = @{ - 'darwin-arm64' = 'e15c1d96c4db70cc834b42693228c6b6bbe7eb575e79d19e8aff2f0bf8ba6e48' - 'darwin-x64' = 'fc534f42b813cf5bf391ee007fe860ac76063e06af3bfc18d5d6256479137cf5' - 'linux-arm64' = '37ab1c50f7269c65c70e5956b7e1b7a2ec9ce8286bb55046de1b15a6b3c9907a' - 'linux-x64' = 'f58f55f94e3559ffd959af9e9f44663a41ec34aef53a1c2d1ff7279ec207af3a' - 'win32-x64' = '492ed40845a7d732f077b575c7ef4d75f06965c991e5e2c4c0e9f356ed0da37f' + 'darwin-arm64' = '648e8909915d8da2b87ae4f4797cd22f2d46d8a7cd025287ab9e6b5436119687' + 'darwin-x64' = 'eac7027d9746eaa61b1d4309e934ef437a01a79e915171704f04539c6275f3f0' + 'linux-arm64' = '4963f4c71dcf05bb8d55e19dd6f1441c5cd5b099e62b30b9b2fbba80cdddaa3c' + 'linux-x64' = 'a8e176fa426360f489bf9c552064faf7e1dd88072b463a13185ff1193a223683' + 'win32-x64' = 'de07306075b5efbf2158c41a7c87fe4939bac45b9b6f953270ed00017a6540be' } $StackBootstrap = @{ 'darwin-arm64' = @{ - manifest = 'b9f750c55834913394ebc459395f5331915d02d98ebeecee3586fb1721a7bc41' - inventory = '3076b6da1fec8727cef3de7187d0ab2818a768e77131952577ebb484d5abd2bc' - bytes = '14937177' - files = '1368' - entries = '1508' + manifest = '15f7138847ef4d13f9c2df455ec04423dc35d30d759d5f952dcb0b69d81db13b' + inventory = 'ecd1dc2a81dbf53803af4669ffb00d31476dced14049e9889beaf4967b7f1dfb' + bytes = '14953559' + files = '1370' + entries = '1510' python = 'python/bin/python3.12' modules = 'modules' } 'darwin-x64' = @{ - manifest = 'c147e254cdf01e8666c69c9e95d6983ea9bbc9c1124f2ad12b3495a4ddc3bd7f' - inventory = '0271d1b25459a9e82ecc6ba99987783650a206ec5280e5818245d47912c03c99' - bytes = '14770496' - files = '1365' - entries = '1504' + manifest = '86fd2f7ad8133e40d507d0a0e09fc860a22108c9d5a6a23d7edd2cdad64318a7' + inventory = '67a3967cf2fb0b062ab0f97b8e7356b85ac28e278fa4d8edd4ef32183d76e1a9' + bytes = '14787792' + files = '1367' + entries = '1506' python = 'python/bin/python3.12' modules = 'modules' } 'linux-arm64' = @{ - manifest = '56ffd15353a55ca0bd9fb847924feada7b9dbd13516ace7f33c6da88a710ad61' - inventory = '8635d2b5765f4bc77582df25099623e9b0cbbe85ae0171f25a395bda6e263dc6' - bytes = '17446785' - files = '4239' - entries = '4423' + manifest = 'e47a45bb030e4d427fcb750df08daa9ccf5f8af6ea67bee395af85ebdf35f455' + inventory = 'e9ed54c5e579d05096f8d8943bcede0488701e5497d468b42e1cc2dd4580a446' + bytes = '17460592' + files = '4241' + entries = '4425' python = 'python/bin/python3.12' modules = 'modules' } 'linux-x64' = @{ - manifest = '60664ab53e44b9967bf21e54a2e2fb263862296537ebd24691ab96e7ed268336' - inventory = '8157380689477b6835695adb0f3bb9e0d0fed5f9bf29586496b8b123088bbed0' - bytes = '20776267' - files = '4239' - entries = '4423' + manifest = 'd36a909ddad7be0de0642766102a5478efd22d7d437c86d8b63ac7bddc24e082' + inventory = '435ed71955a8dc0415cd35579637a35772f44e753e814e34818b38ba60516211' + bytes = '20790291' + files = '4241' + entries = '4425' python = 'python/bin/python3.12' modules = 'modules' } 'win32-x64' = @{ - manifest = 'f17122b311b71a3b37e3207a2c0cdce6f84b3a14e09ef0bceed81be74e90e161' - inventory = 'ce4acb03a685e8e2756656d6d50e491d168c3659ce89882e8f83aff0df2cbc7e' - bytes = '23174114' - files = '3409' - entries = '3676' + manifest = 'c7e2fab80d7aa2bdf52975a0b6e180611830a5d3c013bb302d16ff5c3c0d42dd' + inventory = 'd079da0a54acd521f1b4ace18cea5d4d2a078ff50637266a3982ad43cda0273a' + bytes = '23191788' + files = '3411' + entries = '3678' python = 'python/python.exe' modules = 'modules' } diff --git a/scripts/installs/stack.sh b/scripts/installs/stack.sh index 196ed7e3..f9947752 100755 --- a/scripts/installs/stack.sh +++ b/scripts/installs/stack.sh @@ -2,13 +2,13 @@ set -eu # >>> release pin -STACK_VERSION='0.1.0.dev39' -STACK_SOURCE_COMMIT='9a72159a9c114e323de703fcbf6e030fbf7d480a' -STACK_NPM_VERSION='0.1.0-dev.39' -STACK_KIT_darwin_arm64='e15c1d96c4db70cc834b42693228c6b6bbe7eb575e79d19e8aff2f0bf8ba6e48' -STACK_KIT_darwin_x64='fc534f42b813cf5bf391ee007fe860ac76063e06af3bfc18d5d6256479137cf5' -STACK_KIT_linux_arm64='37ab1c50f7269c65c70e5956b7e1b7a2ec9ce8286bb55046de1b15a6b3c9907a' -STACK_KIT_linux_x64='f58f55f94e3559ffd959af9e9f44663a41ec34aef53a1c2d1ff7279ec207af3a' +STACK_VERSION='0.1.0.dev40' +STACK_SOURCE_COMMIT='a4de3e359d2a4f9b654f9ae7bdf6887c55896b7d' +STACK_NPM_VERSION='0.1.0-dev.40' +STACK_KIT_darwin_arm64='648e8909915d8da2b87ae4f4797cd22f2d46d8a7cd025287ab9e6b5436119687' +STACK_KIT_darwin_x64='eac7027d9746eaa61b1d4309e934ef437a01a79e915171704f04539c6275f3f0' +STACK_KIT_linux_arm64='4963f4c71dcf05bb8d55e19dd6f1441c5cd5b099e62b30b9b2fbba80cdddaa3c' +STACK_KIT_linux_x64='a8e176fa426360f489bf9c552064faf7e1dd88072b463a13185ff1193a223683' # <<< release pin die() { @@ -46,10 +46,10 @@ esac manifest='' inventory='' bytes='' files='' entries='' python='' modules='' # >>> bootstrap targets case "$target" in - darwin-arm64) manifest='b9f750c55834913394ebc459395f5331915d02d98ebeecee3586fb1721a7bc41'; inventory='3076b6da1fec8727cef3de7187d0ab2818a768e77131952577ebb484d5abd2bc'; bytes='14937177'; files='1368'; entries='1508'; python='python/bin/python3.12'; modules='modules' ;; - darwin-x64) manifest='c147e254cdf01e8666c69c9e95d6983ea9bbc9c1124f2ad12b3495a4ddc3bd7f'; inventory='0271d1b25459a9e82ecc6ba99987783650a206ec5280e5818245d47912c03c99'; bytes='14770496'; files='1365'; entries='1504'; python='python/bin/python3.12'; modules='modules' ;; - linux-arm64) manifest='56ffd15353a55ca0bd9fb847924feada7b9dbd13516ace7f33c6da88a710ad61'; inventory='8635d2b5765f4bc77582df25099623e9b0cbbe85ae0171f25a395bda6e263dc6'; bytes='17446785'; files='4239'; entries='4423'; python='python/bin/python3.12'; modules='modules' ;; - linux-x64) manifest='60664ab53e44b9967bf21e54a2e2fb263862296537ebd24691ab96e7ed268336'; inventory='8157380689477b6835695adb0f3bb9e0d0fed5f9bf29586496b8b123088bbed0'; bytes='20776267'; files='4239'; entries='4423'; python='python/bin/python3.12'; modules='modules' ;; + darwin-arm64) manifest='15f7138847ef4d13f9c2df455ec04423dc35d30d759d5f952dcb0b69d81db13b'; inventory='ecd1dc2a81dbf53803af4669ffb00d31476dced14049e9889beaf4967b7f1dfb'; bytes='14953559'; files='1370'; entries='1510'; python='python/bin/python3.12'; modules='modules' ;; + darwin-x64) manifest='86fd2f7ad8133e40d507d0a0e09fc860a22108c9d5a6a23d7edd2cdad64318a7'; inventory='67a3967cf2fb0b062ab0f97b8e7356b85ac28e278fa4d8edd4ef32183d76e1a9'; bytes='14787792'; files='1367'; entries='1506'; python='python/bin/python3.12'; modules='modules' ;; + linux-arm64) manifest='e47a45bb030e4d427fcb750df08daa9ccf5f8af6ea67bee395af85ebdf35f455'; inventory='e9ed54c5e579d05096f8d8943bcede0488701e5497d468b42e1cc2dd4580a446'; bytes='17460592'; files='4241'; entries='4425'; python='python/bin/python3.12'; modules='modules' ;; + linux-x64) manifest='d36a909ddad7be0de0642766102a5478efd22d7d437c86d8b63ac7bddc24e082'; inventory='435ed71955a8dc0415cd35579637a35772f44e753e814e34818b38ba60516211'; bytes='20790291'; files='4241'; entries='4425'; python='python/bin/python3.12'; modules='modules' ;; *) die 'release lacks bootstrap pins' ;; esac # <<< bootstrap targets From 7270430a75c69c8bf06e7412a190de18e1cdd3ab Mon Sep 17 00:00:00 2001 From: Dongdong Chen Date: Thu, 19 Nov 2026 03:27:24 +0800 Subject: [PATCH 2/3] fix(local-installation): end the services a failed install attempt leaves behind A start whose lease identity could not be proved left its supervisor and service running with no record, so PostgreSQL kept postmaster.pid and every Try Again failed until the processes were killed by hand. - When a start fails before its record is kept and the lease cannot be proved, end the process group of the supervisor it spawned and has not reaped yet; that PID cannot belong to another program. - On macOS and Linux, reclaimNativeOrphans now also ends a supervisor whose command line names a job file in the instance's process directory under a lease no record holds, once its environment proves the lease; the services step runs it before PostgreSQL starts, so a state an earlier release left heals on the next attempt. - Split the POSIX stop of one lease out of stopNativeProcess to share it. Co-Authored-By: Claude Opus 5.5 --- packages/local-installation/AGENTS.md | 10 +- .../src/native-process-unverified.test.ts | 71 +++++++++++ .../src/native-process.test.ts | 34 +++++ .../local-installation/src/native-process.ts | 119 ++++++++++++++++-- .../local-installation/src/native-source.ts | 7 +- .../local-installation/src/source-checkout.ts | 2 +- 6 files changed, 229 insertions(+), 14 deletions(-) create mode 100644 packages/local-installation/src/native-process-unverified.test.ts diff --git a/packages/local-installation/AGENTS.md b/packages/local-installation/AGENTS.md index 03e51376..1935e102 100644 --- a/packages/local-installation/AGENTS.md +++ b/packages/local-installation/AGENTS.md @@ -137,7 +137,15 @@ program fails the start after 15 seconds instead of at the deadline. On POSIX, a start that finds the recorded supervisor gone but processes still carrying its lease (the app was killed with it) ends them as a stop does and starts the service again. A live recorded PID or process group -without the lease proves nothing and is refused, never signalled. +without the lease proves nothing and is refused, never signalled. A start +that fails before its record is kept ends what it launched: by the lease, +or, when the lease cannot be proved, by the group of the supervisor it has +not reaped yet, whose PID cannot belong to another program. Before the +services start, `reclaimNativeOrphans` also ends, on POSIX, a supervisor +whose command line names a job file in this instance's process directory +under a lease no record holds, once its environment proves that lease; an +attempt that failed or was killed before keeping a record would otherwise +leave PostgreSQL holding `postmaster.pid` against every retry. Move a freshly downloaded or unpacked file or folder into place with `renameRetrying` from `fs-retry.ts`: on Windows antivirus and indexing hold diff --git a/packages/local-installation/src/native-process-unverified.test.ts b/packages/local-installation/src/native-process-unverified.test.ts new file mode 100644 index 00000000..a5da2d35 --- /dev/null +++ b/packages/local-installation/src/native-process-unverified.test.ts @@ -0,0 +1,71 @@ +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { execFileSync } from "node:child_process"; +import { expect, onTestFinished, test, vi } from "vitest"; + +// Every identity check fails, as a localized `ps` once made them on macOS: +// neither the supervisor nor what it started can be proved by its lease. +vi.mock("./native-process-identity.js", async (original) => ({ + ...(await original()), + nativeProcessIdentity: async () => undefined, + nativeLeaseProcesses: async () => [], +})); +const { startNativeProcess } = await import("./native-process.js"); + +vi.setConfig({ testTimeout: 30_000, expect: { poll: { timeout: 15_000 } } }); + +function alive(pid: number): boolean { + try { + process.kill(pid, 0); + if (process.platform === "linux") + return !execFileSync("cat", [`/proc/${pid}/stat`], { encoding: "utf8" }) + .replace(/^.*\) /s, "") + .startsWith("Z"); + return true; + } catch { + return false; + } +} + +(process.platform === "win32" ? test.skip : test)( + "ends the launched supervisor and its service when their identity cannot be verified", + async () => { + const root = await mkdtemp(path.join(tmpdir(), "native-unverified-")); + onTestFinished(async () => { + try { + execFileSync("pkill", ["-KILL", "-f", root]); + } catch { + // pkill exits 1 when nothing matched. + } + await rm(root, { recursive: true, force: true }); + }); + await expect( + startNativeProcess({ + root, + name: "postgres", + executable: process.execPath, + args: [ + "-e", + "require('node:fs').writeFileSync('service.pid',String(process.pid));setInterval(()=>{},1000)", + // Names the root on its command line, so the check below finds it. + root, + ], + cwd: root, + env: process.env, + }), + ).rejects.toThrow("Native postgres identity could not be verified."); + const service = Number( + await readFile(path.join(root, "service.pid"), "utf8").catch(() => "0"), + ); + // A service left running here held postmaster.pid against every retry. + if (service) await expect.poll(() => alive(service)).toBe(false); + await expect + .poll(() => + execFileSync("ps", ["-axo", "stat=,command="], { encoding: "utf8" }) + .split("\n") + .filter((line) => line.includes(root) && !/^\s*Z/.test(line)), + ) + .toEqual([]); + }, +); diff --git a/packages/local-installation/src/native-process.test.ts b/packages/local-installation/src/native-process.test.ts index 6a46b42b..4f22b8e9 100644 --- a/packages/local-installation/src/native-process.test.ts +++ b/packages/local-installation/src/native-process.test.ts @@ -24,6 +24,7 @@ import { nativeConfigurationChanged, startNativeProcess, inspectNativeProcess, + reclaimNativeOrphans, stopNativeProcess, } from "./native-process.js"; @@ -245,6 +246,39 @@ test("stops descendants that ignore SIGINT after the service leader exits", asyn }, ); +(process.platform === "win32" ? test.skip : test)( + "reclaims a service an earlier attempt left under a lease no record holds", + async (context) => { + const { expect } = context; + const root = await fixture(context); + const service = (name: "web" | "teamhub") => ({ + root, + name, + executable: process.execPath, + args: [ + "-e", + `require('node:fs').writeFileSync('${name}.pid',String(process.pid));setInterval(()=>{},1000)`, + ], + cwd: root, + env: process.env, + }); + const lost = await startNativeProcess(service("web")); + const kept = await startNativeProcess(service("teamhub")); + const pid = async (name: string) => + Number(await readFile(path.join(root, `${name}.pid`), "utf8")); + await expect.poll(() => pid("web").catch(() => 0)).not.toBe(0); + await expect.poll(() => pid("teamhub").catch(() => 0)).not.toBe(0); + // The attempt that started it failed before its record was kept. + await rm(path.join(root, "native/processes/web.json")); + const ended = await reclaimNativeOrphans(root, { ports: [], paths: [] }); + expect(ended).toContain(lost.pid); + await expect.poll(async () => exited(await pid("web"))).toBe(true); + expect(exited(lost.pid!)).toBe(true); + expect(exited(kept.pid!)).toBe(false); + expect(exited(await pid("teamhub"))).toBe(false); + }, +); + test("cleans the launched owned group when identity persistence fails", async (context) => { const { expect } = context; const root = await fixture(context); diff --git a/packages/local-installation/src/native-process.ts b/packages/local-installation/src/native-process.ts index 46e5ac23..489afffd 100644 --- a/packages/local-installation/src/native-process.ts +++ b/packages/local-installation/src/native-process.ts @@ -1,4 +1,4 @@ -import { spawn } from "node:child_process"; +import { spawn, type ChildProcess } from "node:child_process"; import { existsSync } from "node:fs"; import { readFile, rm } from "node:fs/promises"; import { createHash, randomBytes } from "node:crypto"; @@ -396,6 +396,11 @@ export async function startNativeProcess(options: { { cause: error }, ); } + // When the lease could not be proved, the supervisor this call spawned + // and has not reaped is still provably its own, and so is the group it + // leads: end them, or the service keeps its files (postmaster.pid) and + // ports against every retry. + await endUnreapedGroup(child); if (child.connected) child.disconnect(); child.unref(); throw error; @@ -404,6 +409,33 @@ export async function startNativeProcess(options: { } } +/** + * Ends the process group a detached child leads while the child is not yet + * reaped: until then its PID, and so its group, cannot belong to another + * program. + */ +async function endUnreapedGroup(child: ChildProcess): Promise { + if (child.pid === undefined) return; + const exited = new Promise((resolve) => child.once("exit", resolve)); + if (child.exitCode === null && child.signalCode === null) { + try { + process.kill(-child.pid, "SIGKILL"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error; + } + await Promise.race([exited, delay(5_000)]); + } + // The leader's exit does not wait for the rest of its group. + for (let count = 0; count < 50 && (await groupLive(child.pid)); count++) { + try { + process.kill(-child.pid, "SIGKILL"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error; + } + await delay(100); + } +} + /** * Marks a stop refused because the recorded PID now belongs to another * program: none of the service's own processes are left. @@ -417,11 +449,20 @@ export async function stopNativeProcess( ): Promise { const record = await stored(root, name); if (!record) return; - if (process.platform === "win32") { + if (process.platform === "win32") await stopWindowsTree(name, record.lease, record); - await rm(paths(root, name).record, { force: true }); - return; - } + else await stopLease(name, record, interruptMs); + await rm(paths(root, name).record, { force: true }); +} +/** + * macOS and Linux: ends the supervisor holding the lease and everything + * under it, SIGINT first and SIGKILL for what outlives `interruptMs`. + */ +async function stopLease( + name: NativeService, + record: Pick, + interruptMs: number, +): Promise { const current = await observed(record.pid, record.lease); let owned = await nativeLeaseProcesses(record.lease); if (current !== record.birth && !owned.length) { @@ -432,7 +473,6 @@ export async function stopNativeProcess( ), { code: NATIVE_PROCESS_REPLACED }, ); - await rm(paths(root, name).record, { force: true }); return; } const signalOwned = async ( @@ -496,7 +536,6 @@ export async function stopNativeProcess( (await nativeLeaseProcesses(record.lease)).length ) throw new Error(`Native ${name} did not stop; identity record retained.`); - await rm(paths(root, name).record, { force: true }); } function terminate(pid: number) { @@ -664,14 +703,14 @@ export function windowsOrphans( return order; } /** - * Ends what `windowsOrphans` finds for this instance; nothing elsewhere. + * Ends what `windowsOrphans` finds for this instance on Windows, and what + * `reclaimUnrecordedLeases` finds on macOS and Linux; nothing elsewhere. * Returns the PIDs it ended. */ export async function reclaimNativeOrphans( root: string, options: { ports: number[]; paths: string[] }, ): Promise { - if (process.platform !== "win32") return []; const records: Lease[] = []; for (const name of services) try { @@ -680,6 +719,8 @@ export async function reclaimNativeOrphans( } catch { // A damaged record proves nothing; its supervisor counts as unrecorded. } + if (process.platform !== "win32") + return reclaimUnrecordedLeases(root, records); const ended: number[] = []; const deadline = Date.now() + 12_000; for (;;) { @@ -709,3 +750,63 @@ export async function reclaimNativeOrphans( await delay(100); } } + +/** Each process of this user with its command line (macOS and Linux). */ +async function ownCommands(): Promise> { + if (process.platform === "linux") { + const listed = await Promise.all( + (await linuxProcessStates(true)).map(async ({ pid, state }) => { + if (state === "Z") return undefined; + const command = await readFile(`/proc/${pid}/cmdline`, "utf8").catch( + () => "", + ); + return { pid, command: command.split("\0").join(" ") }; + }), + ); + return listed.filter((entry) => entry !== undefined); + } + const result = await runProcess("/bin/ps", ["-axo", "pid=,uid=,command="], { + timeoutMs: 5_000, + }); + if (result.code !== 0) + throw new Error("Native processes could not be enumerated."); + return result.stdout.split("\n").flatMap((line) => { + const match = /^\s*(\d+)\s+(\d+)\s+(.*)$/.exec(line); + return match && Number(match[2]) === process.getuid?.() + ? [{ pid: Number(match[1]), command: match[3]! }] + : []; + }); +} +/** + * macOS and Linux: ends the services an earlier attempt started and kept no + * record of (it failed or was killed before writing one), which otherwise + * keep their files and ports, such as PostgreSQL's postmaster.pid, against + * every retry. A supervisor is this instance's own when its command line + * names a job file in this instance's process directory and its environment + * carries that job's unguessable lease. Returns the supervisors it ended. + */ +async function reclaimUnrecordedLeases( + root: string, + records: Array>, +): Promise { + const recorded = new Set(records.map((record) => record.lease)); + const jobs = path.join(root, "native", "processes") + path.sep; + const ended: number[] = []; + for (const { pid, command } of await ownCommands()) { + if (!command.includes(SUPERVISOR)) continue; + const at = command.indexOf(jobs); + const job = + at < 0 + ? undefined + : /^([a-z-]+)\.([a-f0-9]{64})\.job(?:\s|$)/.exec( + command.slice(at + jobs.length), + ); + if (!job || !services.has(job[1]!) || recorded.has(job[2]!)) continue; + const [, name, lease] = job as unknown as [string, NativeService, string]; + const birth = await observed(pid, lease); + if (!birth) continue; + await stopLease(name, { pid, birth, lease }, 12_000); + ended.push(pid); + } + return ended; +} diff --git a/packages/local-installation/src/native-source.ts b/packages/local-installation/src/native-source.ts index 885f727a..72f058db 100644 --- a/packages/local-installation/src/native-source.ts +++ b/packages/local-installation/src/native-source.ts @@ -1121,9 +1121,10 @@ async function runNativeSource( } } /** - * Windows: ends the processes of this instance an earlier run lost track of - * (an earlier release's stop left them, or their supervisor died), so they - * neither hold the ports nor answer health checks for the new services. + * Ends the processes of this instance an earlier run lost track of (an + * earlier release's stop left them, their supervisor died, or a failed + * attempt kept no record of them), so they neither hold the ports and data + * files nor answer health checks for the new services. */ async function reclaimEarlierProcesses( instance: Instance, diff --git a/packages/local-installation/src/source-checkout.ts b/packages/local-installation/src/source-checkout.ts index 0bf68db6..6f3f7081 100644 --- a/packages/local-installation/src/source-checkout.ts +++ b/packages/local-installation/src/source-checkout.ts @@ -228,7 +228,7 @@ export async function prepareSourceConfiguration( (await portFree(port)) ) continue; - // On Windows the program may be one an earlier run left behind. + // The program may be one an earlier run left behind. await reclaimNativeOrphans(instance.root, { ports: [port], paths: [path.resolve(instance.root, "../.."), instance.root, repo], From 5b9ccf3c7c9454703692a46e103b752acb7cf741 Mon Sep 17 00:00:00 2001 From: Dongdong Chen Date: Thu, 19 Nov 2026 03:27:38 +0800 Subject: [PATCH 3/3] fix(local-installation): apply the approved local model plan again when the stack is not ready A launch that found the local model stack not ready failed with LOCAL_MODELS_RECOVERY_REQUIRED and asked to review an updated plan, which the start failure page cannot do; Try Again failed the same way and Set Up Again was the only way out. A 0.1.3 Pi installed without extension support reports pi-extensions-unavailable from localmodels 0.1.0.dev40 the same way. - When the saved binding for the approved plan is not ready and the saved plan inputs still match, apply that plan again and bind once it verifies; changed inputs still need a new review. The binding's reason codes go out in details.reasons and to the installer log. - The desktop's start failure words the first known reason code and names the action that works for LOCAL_MODELS_RECOVERY_REQUIRED (Try Again) and LOCAL_MODELS_APPROVAL_REQUIRED (Set Up Again). - Map localmodels harness errors to structured failures: a refused harness plan (unusable SIMPLEX_PI_PATH, a Windows runtime root too long for Pi without long paths) is RUNTIME_PLAN_REFUSED, and RUNTIME_ACTION_FAILED and RUNTIME_EXTENSIONS_FAILED keep their codes through dependency preparation and into the failed job's errorCode. Co-Authored-By: Claude Opus 5.5 --- apps/desktop/AGENTS.md | 7 +- apps/desktop/src/main/local-server.test.ts | 59 ++++++++- apps/desktop/src/main/local-server.ts | 54 ++++++++- apps/linger-installer/AGENTS.md | 13 +- apps/linger-installer/src/server/jobs.test.ts | 20 ++++ apps/linger-installer/src/server/jobs.ts | 9 +- .../src/dependencies.test.ts | 39 +++++- .../local-installation/src/dependencies.ts | 10 +- .../src/local-models.test.ts | 112 +++++++++++++++++- .../local-installation/src/local-models.ts | 110 +++++++++++------ .../src/managed-runtimes.test.ts | 61 ++++++++++ .../src/managed-runtimes.ts | 49 ++++++-- 12 files changed, 479 insertions(+), 64 deletions(-) diff --git a/apps/desktop/AGENTS.md b/apps/desktop/AGENTS.md index b82fc3cf..5d73b0b4 100644 --- a/apps/desktop/AGENTS.md +++ b/apps/desktop/AGENTS.md @@ -185,7 +185,12 @@ Gateway and runs Agents. The daemon exits with the app. from their folder, the record keeps naming it, and the start failure page's Retry moves them again. An app older than the recorded version (a downgrade) leaves the services on their newer source and logs - `local_server.source_newer_than_app`. A Quick Install that stopped + `local_server.source_newer_than_app`. When the local model stops a resume, + the start failure names the stack's first known binding reason code + (`details.reasons`, such as `pi-extensions-unavailable`) in its own words + and, by `details.code`, the action that works: Try Again for + `LOCAL_MODELS_RECOVERY_REQUIRED` (the resume applies the approved plan + again), Set Up Again to review a new plan or choose another runtime. A Quick Install that stopped before the computer enrolled is not resumed: setup shows again and Install finishes it, reusing a finished job only while its origin answers (a stopped server gets a new job, which starts the services again). The start failure diff --git a/apps/desktop/src/main/local-server.test.ts b/apps/desktop/src/main/local-server.test.ts index 65b421c0..7e8cf16d 100644 --- a/apps/desktop/src/main/local-server.test.ts +++ b/apps/desktop/src/main/local-server.test.ts @@ -134,7 +134,10 @@ if (args.includes("--resume")) { const repo = args[args.indexOf("--source-repo") + 1]; const servingFile = path.join(home, "resume-serving"); const source = fs.existsSync(servingFile) ? fs.readFileSync(servingFile, "utf8") : fs.existsSync(repo) ? fs.realpathSync(repo) : repo; - process.stdout.write("noise\n" + JSON.stringify({ code, message: code ? "Native teamhub stopped" : "ok", details: { source }, ...(runtime ? { runtime } : {}) }) + "\n"); + // resume-fails may hold the failed outcome's message and details. + const failure = code ? fs.readFileSync(path.join(home, "resume-fails"), "utf8").trim() : ""; + const outcome = failure ? JSON.parse(failure) : { message: code ? "Native teamhub stopped" : "ok", details: { source } }; + process.stdout.write("noise\n" + JSON.stringify({ code, ...outcome, ...(runtime ? { runtime } : {}) }) + "\n"); process.exit(code); } const origin = "http://127.0.0.1:4321"; @@ -2659,6 +2662,60 @@ describe("LocalServerModule after a restart", () => { await f.host.stop("quit"); }); + it.each([ + [ + "LOCAL_MODELS_RECOVERY_REQUIRED", + "Try Again to apply the approved local model plan again, or Set Up Again to review a new plan or choose another AI runtime.", + ], + [ + "LOCAL_MODELS_APPROVAL_REQUIRED", + "Set Up Again to review the local model plan or choose another AI runtime.", + ], + ])( + "names the action that works when the local model stops a restart (%s)", + async (code, action) => { + const f = fixture({}, enrolled, (home) => { + record(home); + writeFileSync( + path.join(home, "resume-fails"), + JSON.stringify({ + message: "The local model stack is not ready.", + details: { code }, + }), + ); + }); + await f.host.start(); + await vi.waitFor(() => expect(f.module.status().phase).toBe("failed")); + expect(f.showStartFailed).toHaveBeenCalledWith( + `The local model stack is not ready. ${action}`, + ); + await f.host.stop("quit"); + }, + ); + + it("says why the local model stack is not ready, from its reason code", async () => { + const f = fixture({}, enrolled, (home) => { + record(home); + writeFileSync( + path.join(home, "resume-fails"), + JSON.stringify({ + message: + "The upstream tool could not verify the local stack binding.", + details: { + code: "LOCAL_MODELS_RECOVERY_REQUIRED", + reasons: ["pi-extensions-unavailable", "gateway-unavailable"], + }, + }), + ); + }); + await f.host.start(); + await vi.waitFor(() => expect(f.module.status().phase).toBe("failed")); + expect(f.showStartFailed).toHaveBeenCalledWith( + "The local model's Pi can't load extensions, which every Linger run needs. The upstream tool could not verify the local stack binding. Try Again to apply the approved local model plan again, or Set Up Again to review a new plan or choose another AI runtime.", + ); + await f.host.stop("quit"); + }); + it("starts the services again when the person tries again", async () => { const f = fixture({}, enrolled, (home) => { record(home); diff --git a/apps/desktop/src/main/local-server.ts b/apps/desktop/src/main/local-server.ts index d69e59d1..fe3c4b11 100644 --- a/apps/desktop/src/main/local-server.ts +++ b/apps/desktop/src/main/local-server.ts @@ -2326,7 +2326,7 @@ export class LocalServerModule implements Module { const outcome = await new Promise<{ code: number; message: string; - details?: { source?: unknown }; + details?: { source?: unknown; code?: unknown; reasons?: unknown }; runtime?: InstanceRuntime; }>((resolve, reject) => { const child = spawn( @@ -2363,7 +2363,7 @@ export class LocalServerModule implements Module { JSON.parse(last) as { code: number; message: string; - details?: { source?: unknown }; + details?: { source?: unknown; code?: unknown; reasons?: unknown }; runtime?: InstanceRuntime; }, ); @@ -2372,8 +2372,29 @@ export class LocalServerModule implements Module { } }); }); - if (outcome.code !== 0) - throw new Error(outcome.message || "The installation stopped"); + if (outcome.code !== 0) { + const { code, reasons } = outcome.details ?? {}; + const action = + typeof code === "string" && Object.hasOwn(RESUME_ACTIONS, code) + ? RESUME_ACTIONS[code] + : undefined; + const reason = Array.isArray(reasons) + ? reasons.find( + (item): item is string => + typeof item === "string" && + Object.hasOwn(LOCAL_MODEL_NOT_READY, item), + ) + : undefined; + throw new Error( + [ + reason && LOCAL_MODEL_NOT_READY[reason], + outcome.message || "The installation stopped", + action, + ] + .filter(Boolean) + .join(" "), + ); + } this.#adoptRuntime(outcome.runtime); const source = outcome.details?.source; return typeof source === "string" ? source : undefined; @@ -2667,6 +2688,31 @@ const LOCAL_MODEL_FAILURES = new Set([ "GPU_SHARING_REVOCATION_FAILED", ]); +// What works from the start-failure page when the local model stops a +// restart: a restart applies the approved plan again, but only setup can +// approve a new one or switch the runtime. +const RESUME_ACTIONS: Record = { + LOCAL_MODELS_RECOVERY_REQUIRED: + "Try Again to apply the approved local model plan again, or Set Up Again to review a new plan or choose another AI runtime.", + LOCAL_MODELS_APPROVAL_REQUIRED: + "Set Up Again to review the local model plan or choose another AI runtime.", +}; + +// Why the local model stack was not ready, by the reason code localmodels' +// setup binding gives. +const LOCAL_MODEL_NOT_READY: Record = { + "pi-extensions-unavailable": + "The local model's Pi can't load extensions, which every Linger run needs.", + "deployment-unavailable": "The local model service isn't running.", + "acceptance-unavailable": "The local model stack hasn't passed its checks.", + "gateway-ownership": "The local model gateway isn't running.", + "gateway-unavailable": "The local model gateway isn't running.", + "pi-model-unavailable": "Pi can't reach the local model.", + "pi-config-changed": "Pi's settings changed while they were checked.", + "plan-invalid": "The approved local model plan can't be read.", + "binding-unavailable": "The local model stack couldn't be checked.", +}; + export interface Reconnect { readonly windowMs: number; readonly maxBackoffMs: number; diff --git a/apps/linger-installer/AGENTS.md b/apps/linger-installer/AGENTS.md index 21e94437..61146daa 100644 --- a/apps/linger-installer/AGENTS.md +++ b/apps/linger-installer/AGENTS.md @@ -28,7 +28,18 @@ once its services are ready on it: until then `instance.json` keeps naming the checkout they run from, the new checkout's migrations run only after the runtime (the local model, for one) is ready, and a run that stops earlier leaves the running services and their schema as they were. A successful -outcome's `details.source` names the checkout the services serve from. With +outcome's `details.source` names the checkout the services serve from. A +local model stack whose saved binding is no longer ready (any setup binding +reason, `pi-extensions-unavailable` from a Pi an earlier localmodels +installed without extension support included) gets its approved plan applied +again when the saved plan inputs still match the current choices; only a +plan the person approved is ever applied, and a stack still not ready ends +`LOCAL_MODELS_RECOVERY_REQUIRED` with the binding's reason codes in +`details.reasons`. A failed job's `errorCode` is the runtime's cause when it +has one (`RUNTIME_PLAN_REFUSED`, a harness plan refused before anything +changed, such as an unusable `SIMPLEX_PI_PATH` or a Windows runtime root too +long for Pi without long paths; `RUNTIME_ACTION_FAILED`; +`RUNTIME_EXTENSIONS_FAILED`), otherwise `INSTALL_FAILED`. With `--external-computer`, a successful outcome carries `runtime`, the same instance runtime as `GET /api/instances/{name}/runtime` (runtime and localmodels tool paths), so the desktop's Computer follows them. diff --git a/apps/linger-installer/src/server/jobs.test.ts b/apps/linger-installer/src/server/jobs.test.ts index 4ce9094f..0b263b9f 100644 --- a/apps/linger-installer/src/server/jobs.test.ts +++ b/apps/linger-installer/src/server/jobs.test.ts @@ -257,6 +257,26 @@ test.each([ }, ); +test.each([ + ["RUNTIME_PLAN_REFUSED", "RUNTIME_PLAN_REFUSED"], + ["RUNTIME_ACTION_FAILED", "RUNTIME_ACTION_FAILED"], + ["RUNTIME_EXTENSIONS_FAILED", "RUNTIME_EXTENSIONS_FAILED"], + ["NATIVE_SOURCE_INSTALLATION_FAILED", "INSTALL_FAILED"], +])( + "a failed installation with runtime cause %s reports it as %s", + async (code, errorCode) => { + const { jobs } = await setup(async () => ({ + code: 5, + state: "failed", + message: "The runtime could not be installed.", + details: { code, stage: "runtime" }, + })); + const job = await jobs.start({ instance: "default" }); + await jobs.wait(job.id); + expect(jobs.get(job.id)).toMatchObject({ state: "failed", errorCode }); + }, +); + const localPlan = { planId: "a".repeat(64), gpuIndex: 1, diff --git a/apps/linger-installer/src/server/jobs.ts b/apps/linger-installer/src/server/jobs.ts index 84165a33..84972d2d 100644 --- a/apps/linger-installer/src/server/jobs.ts +++ b/apps/linger-installer/src/server/jobs.ts @@ -417,7 +417,14 @@ export class JobManager { "LOCAL_MODELS_UNAVAILABLE", "LOCAL_MODELS_APPROVAL_REQUIRED", "LOCAL_MODELS_RECOVERY_REQUIRED", - ].includes(String(details.code)) + ].includes(String(details.code)) || + // Why the runtime could not be installed, in place of INSTALL_FAILED. + (record.job.state === "failed" && + [ + "RUNTIME_PLAN_REFUSED", + "RUNTIME_ACTION_FAILED", + "RUNTIME_EXTENSIONS_FAILED", + ].includes(String(details.code))) ) { record.job.errorCode = String(details.code); if (details.code === "LOCAL_MODELS_APPROVAL_REQUIRED") { diff --git a/packages/local-installation/src/dependencies.test.ts b/packages/local-installation/src/dependencies.test.ts index 26a9d0be..18ac3817 100644 --- a/packages/local-installation/src/dependencies.test.ts +++ b/packages/local-installation/src/dependencies.test.ts @@ -1,5 +1,5 @@ import { afterEach, expect, it, vi } from "vitest"; -import { mkdtemp, rm } from "node:fs/promises"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; import { prepareDependencies } from "./dependencies.js"; @@ -66,3 +66,40 @@ it("returns Runtime timeout category and stage through dependency preparation", details: { code: "COMMAND_TIMEOUT", stage: "runtime" }, }); }); +it.each([ + ["plan", "RUNTIME_ACTION_FAILED", "RUNTIME_PLAN_REFUSED"], + ["apply", "RUNTIME_ACTION_FAILED", "RUNTIME_ACTION_FAILED"], + ["apply", "RUNTIME_EXTENSIONS_FAILED", "RUNTIME_EXTENSIONS_FAILED"], +])( + "returns the harness %s failure %s as %s through dependency preparation", + async (step, upstream, code) => { + const options = { ...(await fixture()), dependency: "pi" as const }; + const localmodels = vi.fn(async (_home: string, args: string[]) => { + if (args[1] === "plan" && step !== "plan") + await writeFile( + args[args.indexOf("--out") + 1]!, + JSON.stringify({ + schema: "localmodels.harness-plan/v1", + id: "a".repeat(64), + runtime: "pi", + }), + ); + const failed = args[1] === step; + return { + code: failed ? 1 : 0, + stdout: failed + ? JSON.stringify({ + schema: "localmodels.harness-error/v1", + code: upstream, + message: "Runtime action failed: upstream cause.", + }) + : "", + stderr: "", + }; + }); + expect(await prepareDependencies(options, { localmodels })).toMatchObject({ + state: "failed", + details: { code, stage: "runtime" }, + }); + }, +); diff --git a/packages/local-installation/src/dependencies.ts b/packages/local-installation/src/dependencies.ts index de9a226a..6ef89773 100644 --- a/packages/local-installation/src/dependencies.ts +++ b/packages/local-installation/src/dependencies.ts @@ -5,6 +5,7 @@ import { privateDirectory } from "./instance.js"; import type { InstallOutcome, InstallerEvent } from "./events.js"; import { installNativeRuntime, + runtimeFailureCode, runtimeVersionFailure, type RuntimeDependencies, } from "./managed-runtimes.js"; @@ -61,12 +62,9 @@ export async function prepareDependencies( emit("failed", message); const code = versionFailure?.code ?? - (error && - typeof error === "object" && - "code" in error && - (error.code === "LOCALMODELS_RUNTIME_UNAVAILABLE" || - error.code === "COMMAND_TIMEOUT") - ? error.code + runtimeFailureCode(error) ?? + ((error as { code?: unknown } | null)?.code === "COMMAND_TIMEOUT" + ? "COMMAND_TIMEOUT" : undefined); return { code: 5, diff --git a/packages/local-installation/src/local-models.test.ts b/packages/local-installation/src/local-models.test.ts index 98dbf118..8a23f16b 100644 --- a/packages/local-installation/src/local-models.test.ts +++ b/packages/local-installation/src/local-models.test.ts @@ -279,7 +279,8 @@ test("replanning requires a separate review and never silently approves", async ).toMatchObject({ details: { code: "LOCAL_MODELS_APPROVAL_REQUIRED" } }); expect(f.call.mock.calls.some((c) => c[1][1] === "apply")).toBe(false); }); -test("saved binding needs fresh upstream health before reuse", async () => { +// A deployed stack whose binding is ready, then observed not ready once. +async function deployedThenNotReady(reason: string) { const f = await fixture(); await prepareLocalModels(f.instance, f.options, { localmodels: f.call }); await prepareLocalModels( @@ -287,11 +288,118 @@ test("saved binding needs fresh upstream health before reuse", async () => { { ...f.options, localModelsApproval: f.id }, { localmodels: f.call }, ); + const ready = structuredClone(f.binding); f.binding.ready = false; f.binding.pi = null as never; + f.binding.reasons = [{ code: reason, message: "upstream text" }] as never; + f.call.mockClear(); + return { ...f, ready }; +} +test.each(["deployment-unavailable", "pi-extensions-unavailable"])( + "a saved binding observed not ready (%s) applies the approved plan again", + async (reason) => { + const f = await deployedThenNotReady(reason); + const original = f.call.getMockImplementation()!; + f.call.mockImplementation(async (...a) => { + // stack apply brings the stack and its Pi back. + if (a[1][1] === "apply") Object.assign(f.binding, f.ready); + return original(...a); + }); + expect( + await prepareLocalModels(f.instance, f.options, { localmodels: f.call }), + ).toBeUndefined(); + const stack = f.call.mock.calls + .map(([, args]) => args) + .filter((args) => args[0] === "stack"); + expect(stack).toEqual([ + [ + "stack", + "apply", + path.join(f.home, "runtime/localmodels/plan.json"), + "--approve", + f.id, + "--jsonl", + ], + ]); + expect( + JSON.parse( + await readFile( + path.join(f.home, "runtime/localmodels/binding.json"), + "utf8", + ), + ), + ).toEqual(f.ready); + expect(f.options.onEvent).toHaveBeenCalledWith( + expect.objectContaining({ + kind: "log", + stage: "runtime", + message: expect.stringContaining(reason), + }), + ); + }, +); +test("a stack still not ready after applying the approved plan again asks for recovery", async () => { + const f = await deployedThenNotReady("gateway-unavailable"); + const result = await prepareLocalModels(f.instance, f.options, { + localmodels: f.call, + }); + expect(result).toMatchObject({ + state: "action_required", + details: { + code: "LOCAL_MODELS_RECOVERY_REQUIRED", + reasons: ["gateway-unavailable"], + }, + }); + expect( + f.call.mock.calls.filter(([, args]) => args[1] === "apply"), + ).toHaveLength(1); + // The earlier ready binding is never replaced by one not verified. + expect( + JSON.parse( + await readFile( + path.join(f.home, "runtime/localmodels/binding.json"), + "utf8", + ), + ), + ).toEqual(f.ready); +}); +test("a failed re-apply of the approved plan reports the upstream cause", async () => { + const f = await deployedThenNotReady("deployment-unavailable"); + const original = f.call.getMockImplementation()!; + f.call.mockImplementation(async (...a) => { + if (a[1][1] !== "apply") return original(...a); + a[2]!.onOutput!("stack: port 8080 is held by another program\n"); + return { code: 1, stdout: "", stderr: "" }; + }); expect( await prepareLocalModels(f.instance, f.options, { localmodels: f.call }), - ).toMatchObject({ details: { code: "LOCAL_MODELS_RECOVERY_REQUIRED" } }); + ).toMatchObject({ + state: "action_required", + // What made it apply the plan again. + details: { + code: "LOCAL_MODELS_RECOVERY_REQUIRED", + reasons: ["deployment-unavailable"], + }, + message: expect.stringContaining( + "stack: port 8080 is held by another program", + ), + }); +}); +test("a not-ready binding is never re-applied once resource choices changed", async () => { + const f = await deployedThenNotReady("deployment-unavailable"); + const result = await prepareLocalModels( + f.instance, + { ...f.options, shareGpu: false }, + { localmodels: f.call }, + ); + expect(result).toMatchObject({ + details: { code: "LOCAL_MODELS_RECOVERY_REQUIRED" }, + }); + expect( + f.call.mock.calls.some( + ([, args]) => args[0] === "stack" && ["plan", "apply"].includes(args[1]!), + ), + ).toBe(false); }); test("cancellation propagates without pretending to finish", async () => { const f = await fixture(); diff --git a/packages/local-installation/src/local-models.ts b/packages/local-installation/src/local-models.ts index cabc2020..c1fac212 100644 --- a/packages/local-installation/src/local-models.ts +++ b/packages/local-installation/src/local-models.ts @@ -39,6 +39,8 @@ interface Binding { environment: Record; configFingerprint: string; } | null; + // Why the stack is not ready, such as pi-extensions-unavailable. + reasons?: { code?: unknown; message?: unknown }[]; } interface Options { shareGpu?: boolean; @@ -182,6 +184,22 @@ function binding(value: unknown, id?: string): Binding { throw new Error("Invalid upstream Pi binding."); return b; } +// The binding's reason codes, such as pi-extensions-unavailable. +function reasonCodes(observed: Binding | null): string[] { + return (observed?.reasons ?? []).flatMap((reason) => + typeof reason?.code === "string" && + /^[a-z][a-z0-9-]{0,63}$/.test(reason.code) + ? [reason.code] + : [], + ); +} +// The first reason's upstream message, for display only. +function reasonText(observed: Binding | null): string { + const message = observed?.reasons?.[0]?.message; + return typeof message === "string" && message.trim() + ? `: ${message.trim().replace(/\s+/g, " ").slice(0, 300)}` + : ""; +} export async function localModelsEnvironment( instance: Instance, ): Promise { @@ -235,6 +253,8 @@ export async function prepareLocalModels( dependencies: Dependencies = {}, ): Promise { options.signal.throwIfAborted(); + // Why the binding was not ready, for the client to word; set once observed. + let reasons: string[] = []; const action = ( code: string, message: string, @@ -243,7 +263,11 @@ export async function prepareLocalModels( code: 4, state: "action_required", message, - details: { code, ...(summary ? { localModelsPlan: summary } : {}) }, + details: { + code, + ...(summary ? { localModelsPlan: summary } : {}), + ...(reasons.length ? { reasons } : {}), + }, }); if (options.localModelsReplan && options.localModelsApproval) return action( @@ -399,23 +423,10 @@ export async function prepareLocalModels( "--json", ]); if (result.code !== 0) return null; - return binding(JSON.parse(result.stdout), summary.planId); + const observed = binding(JSON.parse(result.stdout), summary.planId); + reasons = observed.ready ? [] : reasonCodes(observed); + return observed; }; - if ( - previous && - !options.localModelsReplan && - binding(JSON.parse(previous)).planId === summary.planId - ) { - const observed = await observe(); - if (observed?.ready) { - await atomicPrivate(bindFile, JSON.stringify(observed)); - return; - } - return action( - "LOCAL_MODELS_RECOVERY_REQUIRED", - "The upstream stack is not ready. Review an updated plan; existing resources were preserved.", - ); - } let inputs: | { planId?: unknown; shareGpu?: unknown; decision?: unknown } | undefined; @@ -425,24 +436,51 @@ export async function prepareLocalModels( } catch { // A damaged or incomplete approval binding cannot authorize deployment. } - if ( - inputs?.planId !== summary.planId || - inputs?.shareGpu !== shareGpu || - inputs?.decision !== false - ) - return action( - "LOCAL_MODELS_RECOVERY_REQUIRED", - "Local model resource choices changed or their approval binding is unavailable. Review a new plan; existing resources were preserved.", - ); - if ( - options.localModelsApproval !== summary.planId || - options.localModelsReplan - ) - return action( - "LOCAL_MODELS_APPROVAL_REQUIRED", - "Review the upstream local model plan before installing.", - summary, - ); + const approvedInputs = + inputs?.planId === summary.planId && + inputs?.shareGpu === shareGpu && + inputs?.decision === false; + // A binding saved for this plan means it was approved and deployed once. + const deployed = + !!previous && + !options.localModelsReplan && + binding(JSON.parse(previous)).planId === summary.planId; + if (deployed) { + const observed = await observe(); + if (observed?.ready) { + await atomicPrivate(bindFile, JSON.stringify(observed)); + return; + } + if (!approvedInputs) + return action( + "LOCAL_MODELS_RECOVERY_REQUIRED", + "The local model stack is not ready, and its resource choices changed since its plan was approved. Review a new plan; existing resources were preserved.", + ); + // Whatever stopped the approved stack (a stopped or adopted + // deployment, a Pi without extension support from an earlier + // localmodels), applying the same plan again restores it; the + // approval it was deployed under still holds. + options.onEvent({ + kind: "log", + stage: "runtime", + message: `The local model stack is not ready (${reasons.join(", ") || "unknown"}); applying its approved plan again.\n`, + }); + } else { + if (!approvedInputs) + return action( + "LOCAL_MODELS_RECOVERY_REQUIRED", + "Local model resource choices changed or their approval binding is unavailable. Review a new plan; existing resources were preserved.", + ); + if ( + options.localModelsApproval !== summary.planId || + options.localModelsReplan + ) + return action( + "LOCAL_MODELS_APPROVAL_REQUIRED", + "Review the upstream local model plan before installing.", + summary, + ); + } const applied = await command([ "stack", "apply", @@ -462,7 +500,7 @@ export async function prepareLocalModels( if (!observed?.ready) return action( "LOCAL_MODELS_RECOVERY_REQUIRED", - "The upstream tool could not verify the local stack binding. Existing resources were preserved.", + `The upstream tool could not verify the local stack binding${reasonText(observed)}. Existing resources were preserved.`, ); await atomicPrivate(bindFile, JSON.stringify(observed)); options.onEvent({ diff --git a/packages/local-installation/src/managed-runtimes.test.ts b/packages/local-installation/src/managed-runtimes.test.ts index 0423a51e..6a2c6653 100644 --- a/packages/local-installation/src/managed-runtimes.test.ts +++ b/packages/local-installation/src/managed-runtimes.test.ts @@ -371,3 +371,64 @@ it("does not retry once the install is cancelled during the wait", async () => { ).rejects.toThrow(); expect(applies(localmodels)).toHaveLength(1); }); + +// A harness command that fails at `step` with one harness-error/v1 line. +function harnessFailing(step: "plan" | "apply", code: string, message = "") { + return vi.fn(async (_home: string, args: string[]) => { + if (args[1] === "plan" && step !== "plan") + await writeFile( + args[args.indexOf("--out") + 1]!, + JSON.stringify({ + schema: "localmodels.harness-plan/v1", + id: "a".repeat(64), + runtime: "pi", + }), + ); + const failed = args[1] === step; + return { + code: failed ? 1 : 0, + stdout: failed + ? JSON.stringify({ + schema: "localmodels.harness-error/v1", + code, + message, + }) + : "", + stderr: "", + }; + }); +} + +it("reports a plan the harness refused, such as a Windows runtime root too long for Pi", async () => { + const options = await fixture(); + const localmodels = harnessFailing( + "plan", + "RUNTIME_ACTION_FAILED", + "Runtime action failed: The runtime root is 120 characters; Pi's dependency tree fits below a root of at most 90 on Windows unless long paths are enabled. Inspect the plan and retry.", + ); + const failure = await installNativeRuntime("pi", options, { + localmodels, + }).catch((error: Error & { code: string }) => error); + expect(failure).toMatchObject({ code: "RUNTIME_PLAN_REFUSED" }); + expect((failure as Error).message).toContain( + "The runtime root is 120 characters", + ); + expect(localmodels.mock.calls.some(([, args]) => args[1] === "apply")).toBe( + false, + ); +}); + +it.each(["RUNTIME_ACTION_FAILED", "RUNTIME_EXTENSIONS_FAILED"])( + "keeps the harness apply failure %s as its code", + async (code) => { + const options = await fixture(); + const failure = await installNativeRuntime("pi", options, { + localmodels: harnessFailing( + "apply", + code, + "Runtime action failed: pi is still not usable after apply. Inspect the plan and retry.", + ), + }).catch((error: Error & { code: string }) => error); + expect(failure).toMatchObject({ code }); + }, +); diff --git a/packages/local-installation/src/managed-runtimes.ts b/packages/local-installation/src/managed-runtimes.ts index 634c5fa1..54733872 100644 --- a/packages/local-installation/src/managed-runtimes.ts +++ b/packages/local-installation/src/managed-runtimes.ts @@ -53,6 +53,25 @@ export function runtimeVersionFailure( ? { code, message: versionFailures[code]! } : undefined; } +/** + * The failures this module reports with their own code: a harness plan + * refused before anything changed (an unusable SIMPLEX_PI_PATH, a Windows + * runtime root too long for Pi without long paths), an apply that left the + * runtime unusable, a Pi that cannot load extensions, and any other + * localmodels failure. + */ +const runtimeFailures = new Set([ + "RUNTIME_PLAN_REFUSED", + "RUNTIME_ACTION_FAILED", + "RUNTIME_EXTENSIONS_FAILED", + "LOCALMODELS_RUNTIME_UNAVAILABLE", +]); +export function runtimeFailureCode(error: unknown): string | undefined { + const code = (error as { code?: unknown } | null)?.code; + return typeof code === "string" && runtimeFailures.has(code) + ? code + : undefined; +} type LocalmodelsResult = Awaited>; function unavailable(failed?: { args: string[]; @@ -75,9 +94,25 @@ function unavailable(failed?: { } catch { /* Non-JSON output is read below as redacted diagnostics. */ } - const { cause, log } = localmodelsFailure(failed.args, failed.result); + const { code, cause, log } = localmodelsFailure(failed.args, failed.result); for (const message of log) failed.onEvent?.({ kind: "log", stage: "runtime", message }); + if (code === "RUNTIME_EXTENSIONS_FAILED") + return Object.assign( + new Error( + "Pi cannot load extensions, which Linger needs on every run. If you installed Pi yourself, update or remove it, then try again.", + ), + { code }, + ); + if (code === "RUNTIME_ACTION_FAILED") { + const planned = failed.args[1] === "plan"; + return Object.assign( + new Error( + `The localmodels Runtime ${planned ? "plan was refused" : "operation failed"}${cause ? `: ${cause}` : "."}`, + ), + { code: planned ? "RUNTIME_PLAN_REFUSED" : code }, + ); + } if (cause) return Object.assign( new Error(`The localmodels Runtime operation failed: ${cause}`), @@ -141,11 +176,7 @@ export async function inspectRuntimeDescriptors( const failure = runtimeVersionFailure(error); if (failure) throw Object.assign(new Error(failure.message), { code: failure.code }); - if ( - error instanceof Error && - (error as { code?: string }).code === "LOCALMODELS_RUNTIME_UNAVAILABLE" - ) - throw error; + if (error instanceof Error && runtimeFailureCode(error)) throw error; throw unavailable(); } } @@ -296,11 +327,7 @@ export async function installNativeRuntime( const failure = runtimeVersionFailure(error); if (failure) throw Object.assign(new Error(failure.message), { code: failure.code }); - if ( - error instanceof Error && - (error as { code?: string }).code === "LOCALMODELS_RUNTIME_UNAVAILABLE" - ) - throw error; + if (error instanceof Error && runtimeFailureCode(error)) throw error; throw unavailable(); } finally { await rm(temporary, { recursive: true, force: true });