Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion apps/desktop/AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -185,7 +185,12 @@ Gateway and runs Agents. The daemon exits with the app.
from their folder, the record keeps naming it, and the start failure page's
Retry moves them again. An app older than the recorded version
(a downgrade) leaves the services on their newer source and logs
`local_server.source_newer_than_app`. A Quick Install that stopped
`local_server.source_newer_than_app`. When the local model stops a resume,
the start failure names the stack's first known binding reason code
(`details.reasons`, such as `pi-extensions-unavailable`) in its own words
and, by `details.code`, the action that works: Try Again for
`LOCAL_MODELS_RECOVERY_REQUIRED` (the resume applies the approved plan
again), Set Up Again to review a new plan or choose another runtime. A Quick Install that stopped
before the computer enrolled is not resumed: setup shows again and Install
finishes it, reusing a finished job only while its origin answers (a stopped
server gets a new job, which starts the services again). The start failure
Expand Down
10 changes: 5 additions & 5 deletions apps/desktop/distribution/darwin-arm64.json
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,11 @@
"platform": "darwin",
"arch": "arm64",
"localmodelsTool": {
"url": "https://d2ikbbcl3j6jz9.cloudfront.net/cli/localmodels/0.1.0.dev39/darwin-arm64.tgz",
"sha256": "e15c1d96c4db70cc834b42693228c6b6bbe7eb575e79d19e8aff2f0bf8ba6e48",
"bytes": 14937177,
"sourceCommit": "9a72159a9c114e323de703fcbf6e030fbf7d480a",
"manifestSha256": "b9f750c55834913394ebc459395f5331915d02d98ebeecee3586fb1721a7bc41"
"url": "https://d2ikbbcl3j6jz9.cloudfront.net/cli/localmodels/0.1.0.dev40/darwin-arm64.tgz",
"sha256": "648e8909915d8da2b87ae4f4797cd22f2d46d8a7cd025287ab9e6b5436119687",
"bytes": 14953559,
"sourceCommit": "a4de3e359d2a4f9b654f9ae7bdf6887c55896b7d",
"manifestSha256": "15f7138847ef4d13f9c2df455ec04423dc35d30d759d5f952dcb0b69d81db13b"
},
"nativeSources": [
{
Expand Down
10 changes: 5 additions & 5 deletions apps/desktop/distribution/linux-x64.json
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,11 @@
"platform": "linux",
"arch": "x64",
"localmodelsTool": {
"url": "https://d2ikbbcl3j6jz9.cloudfront.net/cli/localmodels/0.1.0.dev39/linux-x64.tgz",
"sha256": "f58f55f94e3559ffd959af9e9f44663a41ec34aef53a1c2d1ff7279ec207af3a",
"bytes": 20776267,
"sourceCommit": "9a72159a9c114e323de703fcbf6e030fbf7d480a",
"manifestSha256": "60664ab53e44b9967bf21e54a2e2fb263862296537ebd24691ab96e7ed268336"
"url": "https://d2ikbbcl3j6jz9.cloudfront.net/cli/localmodels/0.1.0.dev40/linux-x64.tgz",
"sha256": "a8e176fa426360f489bf9c552064faf7e1dd88072b463a13185ff1193a223683",
"bytes": 20790291,
"sourceCommit": "a4de3e359d2a4f9b654f9ae7bdf6887c55896b7d",
"manifestSha256": "d36a909ddad7be0de0642766102a5478efd22d7d437c86d8b63ac7bddc24e082"
},
"nativeSources": [
{
Expand Down
10 changes: 5 additions & 5 deletions apps/desktop/distribution/win32-x64.json
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,11 @@
"platform": "win32",
"arch": "x64",
"localmodelsTool": {
"url": "https://d2ikbbcl3j6jz9.cloudfront.net/cli/localmodels/0.1.0.dev39/win32-x64.tgz",
"sha256": "492ed40845a7d732f077b575c7ef4d75f06965c991e5e2c4c0e9f356ed0da37f",
"bytes": 23174114,
"sourceCommit": "9a72159a9c114e323de703fcbf6e030fbf7d480a",
"manifestSha256": "f17122b311b71a3b37e3207a2c0cdce6f84b3a14e09ef0bceed81be74e90e161"
"url": "https://d2ikbbcl3j6jz9.cloudfront.net/cli/localmodels/0.1.0.dev40/win32-x64.tgz",
"sha256": "de07306075b5efbf2158c41a7c87fe4939bac45b9b6f953270ed00017a6540be",
"bytes": 23191788,
"sourceCommit": "a4de3e359d2a4f9b654f9ae7bdf6887c55896b7d",
"manifestSha256": "c7e2fab80d7aa2bdf52975a0b6e180611830a5d3c013bb302d16ff5c3c0d42dd"
},
"nativeSources": [
{
Expand Down
59 changes: 58 additions & 1 deletion apps/desktop/src/main/local-server.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,10 @@ if (args.includes("--resume")) {
const repo = args[args.indexOf("--source-repo") + 1];
const servingFile = path.join(home, "resume-serving");
const source = fs.existsSync(servingFile) ? fs.readFileSync(servingFile, "utf8") : fs.existsSync(repo) ? fs.realpathSync(repo) : repo;
process.stdout.write("noise\n" + JSON.stringify({ code, message: code ? "Native teamhub stopped" : "ok", details: { source }, ...(runtime ? { runtime } : {}) }) + "\n");
// resume-fails may hold the failed outcome's message and details.
const failure = code ? fs.readFileSync(path.join(home, "resume-fails"), "utf8").trim() : "";
const outcome = failure ? JSON.parse(failure) : { message: code ? "Native teamhub stopped" : "ok", details: { source } };
process.stdout.write("noise\n" + JSON.stringify({ code, ...outcome, ...(runtime ? { runtime } : {}) }) + "\n");
process.exit(code);
}
const origin = "http://127.0.0.1:4321";
Expand Down Expand Up @@ -2659,6 +2662,60 @@ describe("LocalServerModule after a restart", () => {
await f.host.stop("quit");
});

it.each([
[
"LOCAL_MODELS_RECOVERY_REQUIRED",
"Try Again to apply the approved local model plan again, or Set Up Again to review a new plan or choose another AI runtime.",
],
[
"LOCAL_MODELS_APPROVAL_REQUIRED",
"Set Up Again to review the local model plan or choose another AI runtime.",
],
])(
"names the action that works when the local model stops a restart (%s)",
async (code, action) => {
const f = fixture({}, enrolled, (home) => {
record(home);
writeFileSync(
path.join(home, "resume-fails"),
JSON.stringify({
message: "The local model stack is not ready.",
details: { code },
}),
);
});
await f.host.start();
await vi.waitFor(() => expect(f.module.status().phase).toBe("failed"));
expect(f.showStartFailed).toHaveBeenCalledWith(
`The local model stack is not ready. ${action}`,
);
await f.host.stop("quit");
},
);

it("says why the local model stack is not ready, from its reason code", async () => {
const f = fixture({}, enrolled, (home) => {
record(home);
writeFileSync(
path.join(home, "resume-fails"),
JSON.stringify({
message:
"The upstream tool could not verify the local stack binding.",
details: {
code: "LOCAL_MODELS_RECOVERY_REQUIRED",
reasons: ["pi-extensions-unavailable", "gateway-unavailable"],
},
}),
);
});
await f.host.start();
await vi.waitFor(() => expect(f.module.status().phase).toBe("failed"));
expect(f.showStartFailed).toHaveBeenCalledWith(
"The local model's Pi can't load extensions, which every Linger run needs. The upstream tool could not verify the local stack binding. Try Again to apply the approved local model plan again, or Set Up Again to review a new plan or choose another AI runtime.",
);
await f.host.stop("quit");
});

it("starts the services again when the person tries again", async () => {
const f = fixture({}, enrolled, (home) => {
record(home);
Expand Down
54 changes: 50 additions & 4 deletions apps/desktop/src/main/local-server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2326,7 +2326,7 @@ export class LocalServerModule implements Module {
const outcome = await new Promise<{
code: number;
message: string;
details?: { source?: unknown };
details?: { source?: unknown; code?: unknown; reasons?: unknown };
runtime?: InstanceRuntime;
}>((resolve, reject) => {
const child = spawn(
Expand Down Expand Up @@ -2363,7 +2363,7 @@ export class LocalServerModule implements Module {
JSON.parse(last) as {
code: number;
message: string;
details?: { source?: unknown };
details?: { source?: unknown; code?: unknown; reasons?: unknown };
runtime?: InstanceRuntime;
},
);
Expand All @@ -2372,8 +2372,29 @@ export class LocalServerModule implements Module {
}
});
});
if (outcome.code !== 0)
throw new Error(outcome.message || "The installation stopped");
if (outcome.code !== 0) {
const { code, reasons } = outcome.details ?? {};
const action =
typeof code === "string" && Object.hasOwn(RESUME_ACTIONS, code)
? RESUME_ACTIONS[code]
: undefined;
const reason = Array.isArray(reasons)
? reasons.find(
(item): item is string =>
typeof item === "string" &&
Object.hasOwn(LOCAL_MODEL_NOT_READY, item),
)
: undefined;
throw new Error(
[
reason && LOCAL_MODEL_NOT_READY[reason],
outcome.message || "The installation stopped",
action,
]
.filter(Boolean)
.join(" "),
);
}
this.#adoptRuntime(outcome.runtime);
const source = outcome.details?.source;
return typeof source === "string" ? source : undefined;
Expand Down Expand Up @@ -2667,6 +2688,31 @@ const LOCAL_MODEL_FAILURES = new Set([
"GPU_SHARING_REVOCATION_FAILED",
]);

// What works from the start-failure page when the local model stops a
// restart: a restart applies the approved plan again, but only setup can
// approve a new one or switch the runtime.
const RESUME_ACTIONS: Record<string, string> = {
LOCAL_MODELS_RECOVERY_REQUIRED:
"Try Again to apply the approved local model plan again, or Set Up Again to review a new plan or choose another AI runtime.",
LOCAL_MODELS_APPROVAL_REQUIRED:
"Set Up Again to review the local model plan or choose another AI runtime.",
};

// Why the local model stack was not ready, by the reason code localmodels'
// setup binding gives.
const LOCAL_MODEL_NOT_READY: Record<string, string> = {
"pi-extensions-unavailable":
"The local model's Pi can't load extensions, which every Linger run needs.",
"deployment-unavailable": "The local model service isn't running.",
"acceptance-unavailable": "The local model stack hasn't passed its checks.",
"gateway-ownership": "The local model gateway isn't running.",
"gateway-unavailable": "The local model gateway isn't running.",
"pi-model-unavailable": "Pi can't reach the local model.",
"pi-config-changed": "Pi's settings changed while they were checked.",
"plan-invalid": "The approved local model plan can't be read.",
"binding-unavailable": "The local model stack couldn't be checked.",
};

export interface Reconnect {
readonly windowMs: number;
readonly maxBackoffMs: number;
Expand Down
13 changes: 12 additions & 1 deletion apps/linger-installer/AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,18 @@ once its services are ready on it: until then `instance.json` keeps naming the
checkout they run from, the new checkout's migrations run only after the
runtime (the local model, for one) is ready, and a run that stops earlier
leaves the running services and their schema as they were. A successful
outcome's `details.source` names the checkout the services serve from. With
outcome's `details.source` names the checkout the services serve from. A
local model stack whose saved binding is no longer ready (any setup binding
reason, `pi-extensions-unavailable` from a Pi an earlier localmodels
installed without extension support included) gets its approved plan applied
again when the saved plan inputs still match the current choices; only a
plan the person approved is ever applied, and a stack still not ready ends
`LOCAL_MODELS_RECOVERY_REQUIRED` with the binding's reason codes in
`details.reasons`. A failed job's `errorCode` is the runtime's cause when it
has one (`RUNTIME_PLAN_REFUSED`, a harness plan refused before anything
changed, such as an unusable `SIMPLEX_PI_PATH` or a Windows runtime root too
long for Pi without long paths; `RUNTIME_ACTION_FAILED`;
`RUNTIME_EXTENSIONS_FAILED`), otherwise `INSTALL_FAILED`. With
`--external-computer`, a successful outcome carries `runtime`, the same instance runtime as
`GET /api/instances/{name}/runtime` (runtime and localmodels tool paths), so
the desktop's Computer follows them.
Expand Down
20 changes: 20 additions & 0 deletions apps/linger-installer/src/server/jobs.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -257,6 +257,26 @@ test.each([
},
);

test.each([
["RUNTIME_PLAN_REFUSED", "RUNTIME_PLAN_REFUSED"],
["RUNTIME_ACTION_FAILED", "RUNTIME_ACTION_FAILED"],
["RUNTIME_EXTENSIONS_FAILED", "RUNTIME_EXTENSIONS_FAILED"],
["NATIVE_SOURCE_INSTALLATION_FAILED", "INSTALL_FAILED"],
])(
"a failed installation with runtime cause %s reports it as %s",
async (code, errorCode) => {
const { jobs } = await setup(async () => ({
code: 5,
state: "failed",
message: "The runtime could not be installed.",
details: { code, stage: "runtime" },
}));
const job = await jobs.start({ instance: "default" });
await jobs.wait(job.id);
expect(jobs.get(job.id)).toMatchObject({ state: "failed", errorCode });
},
);

const localPlan = {
planId: "a".repeat(64),
gpuIndex: 1,
Expand Down
9 changes: 8 additions & 1 deletion apps/linger-installer/src/server/jobs.ts
Original file line number Diff line number Diff line change
Expand Up @@ -417,7 +417,14 @@ export class JobManager {
"LOCAL_MODELS_UNAVAILABLE",
"LOCAL_MODELS_APPROVAL_REQUIRED",
"LOCAL_MODELS_RECOVERY_REQUIRED",
].includes(String(details.code))
].includes(String(details.code)) ||
// Why the runtime could not be installed, in place of INSTALL_FAILED.
(record.job.state === "failed" &&
[
"RUNTIME_PLAN_REFUSED",
"RUNTIME_ACTION_FAILED",
"RUNTIME_EXTENSIONS_FAILED",
].includes(String(details.code)))
) {
record.job.errorCode = String(details.code);
if (details.code === "LOCAL_MODELS_APPROVAL_REQUIRED") {
Expand Down
10 changes: 9 additions & 1 deletion packages/local-installation/AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,15 @@ program fails the start after 15 seconds instead of at the deadline.
On POSIX, a start that finds the recorded supervisor gone but processes
still carrying its lease (the app was killed with it) ends them as a stop
does and starts the service again. A live recorded PID or process group
without the lease proves nothing and is refused, never signalled.
without the lease proves nothing and is refused, never signalled. A start
that fails before its record is kept ends what it launched: by the lease,
or, when the lease cannot be proved, by the group of the supervisor it has
not reaped yet, whose PID cannot belong to another program. Before the
services start, `reclaimNativeOrphans` also ends, on POSIX, a supervisor
whose command line names a job file in this instance's process directory
under a lease no record holds, once its environment proves that lease; an
attempt that failed or was killed before keeping a record would otherwise
leave PostgreSQL holding `postmaster.pid` against every retry.

Move a freshly downloaded or unpacked file or folder into place with
`renameRetrying` from `fs-retry.ts`: on Windows antivirus and indexing hold
Expand Down
39 changes: 38 additions & 1 deletion packages/local-installation/src/dependencies.test.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { afterEach, expect, it, vi } from "vitest";
import { mkdtemp, rm } from "node:fs/promises";
import { mkdtemp, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import path from "node:path";
import { prepareDependencies } from "./dependencies.js";
Expand Down Expand Up @@ -66,3 +66,40 @@ it("returns Runtime timeout category and stage through dependency preparation",
details: { code: "COMMAND_TIMEOUT", stage: "runtime" },
});
});
it.each([
["plan", "RUNTIME_ACTION_FAILED", "RUNTIME_PLAN_REFUSED"],
["apply", "RUNTIME_ACTION_FAILED", "RUNTIME_ACTION_FAILED"],
["apply", "RUNTIME_EXTENSIONS_FAILED", "RUNTIME_EXTENSIONS_FAILED"],
])(
"returns the harness %s failure %s as %s through dependency preparation",
async (step, upstream, code) => {
const options = { ...(await fixture()), dependency: "pi" as const };
const localmodels = vi.fn(async (_home: string, args: string[]) => {
if (args[1] === "plan" && step !== "plan")
await writeFile(
args[args.indexOf("--out") + 1]!,
JSON.stringify({
schema: "localmodels.harness-plan/v1",
id: "a".repeat(64),
runtime: "pi",
}),
);
const failed = args[1] === step;
return {
code: failed ? 1 : 0,
stdout: failed
? JSON.stringify({
schema: "localmodels.harness-error/v1",
code: upstream,
message: "Runtime action failed: upstream cause.",
})
: "",
stderr: "",
};
});
expect(await prepareDependencies(options, { localmodels })).toMatchObject({
state: "failed",
details: { code, stage: "runtime" },
});
},
);
10 changes: 4 additions & 6 deletions packages/local-installation/src/dependencies.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import { privateDirectory } from "./instance.js";
import type { InstallOutcome, InstallerEvent } from "./events.js";
import {
installNativeRuntime,
runtimeFailureCode,
runtimeVersionFailure,
type RuntimeDependencies,
} from "./managed-runtimes.js";
Expand Down Expand Up @@ -61,12 +62,9 @@ export async function prepareDependencies(
emit("failed", message);
const code =
versionFailure?.code ??
(error &&
typeof error === "object" &&
"code" in error &&
(error.code === "LOCALMODELS_RUNTIME_UNAVAILABLE" ||
error.code === "COMMAND_TIMEOUT")
? error.code
runtimeFailureCode(error) ??
((error as { code?: unknown } | null)?.code === "COMMAND_TIMEOUT"
? "COMMAND_TIMEOUT"
: undefined);
return {
code: 5,
Expand Down
Loading
Loading