From ec77e1c60b6edfbd0b489d02a733c3fa57525ea1 Mon Sep 17 00:00:00 2001 From: Dongdong Chen Date: Wed, 18 Nov 2026 21:56:33 +0800 Subject: [PATCH] feat(desktop): reach the update feed from mainland China through npm Task: whi-753-updater (Refs https://linear.app/linger1024/issue/WHI-753) --- .github/workflows/desktop-release.yml | 17 + apps/desktop/package.json | 1 + apps/desktop/src/main/index.ts | 1 + apps/desktop/src/main/test-support.ts | 36 ++ apps/desktop/src/main/update-manifest.test.ts | 34 ++ apps/desktop/src/main/update-manifest.ts | 23 +- apps/desktop/src/main/update-npm.test.ts | 154 +++++++ apps/desktop/src/main/update-npm.ts | 172 ++++++++ apps/desktop/src/main/updater.test.ts | 346 ++++++++++++++- apps/desktop/src/main/updater.ts | 401 +++++++++++++++++- docs/desktop-release.md | 110 ++++- pnpm-lock.yaml | 3 + scripts/ci/desktop-npm-pack.mjs | 141 ++++++ scripts/ci/desktop-release.sh | 104 ++++- scripts/ci/desktop-release.test.ts | 238 +++++++++++ 15 files changed, 1723 insertions(+), 58 deletions(-) create mode 100644 apps/desktop/src/main/update-npm.test.ts create mode 100644 apps/desktop/src/main/update-npm.ts create mode 100644 scripts/ci/desktop-npm-pack.mjs diff --git a/.github/workflows/desktop-release.yml b/.github/workflows/desktop-release.yml index b1f2bd1be..f4a9fe7d8 100644 --- a/.github/workflows/desktop-release.yml +++ b/.github/workflows/desktop-release.yml @@ -223,3 +223,20 @@ jobs: run: >- scripts/ci/desktop-release.sh github-release release "${{ needs.prepare.outputs.tag }}" "${{ needs.prepare.outputs.version }}" + # The China update feed: the signed manifest and each package's parts + # as npm packages, which npmmirror serves. Without NPM_TOKEN or a + # manifest nothing goes to npm; apps in China, finding nothing newer + # there, then take this version from the GitHub release. + - name: Publish the npm update feed + env: + NPM_TOKEN: ${{ secrets.NPM_TOKEN }} + VERSION: ${{ needs.prepare.outputs.version }} + shell: bash + run: | + set -euo pipefail + if [ -z "$NPM_TOKEN" ] || [ ! -f release/manifest.json ]; then + echo "::warning::No NPM_TOKEN or no update manifest: $VERSION is not published to npm (docs/desktop-release.md#china-update-feed)." + exit 0 + fi + scripts/ci/desktop-release.sh npm-pack release "$VERSION" "$RUNNER_TEMP/npm" + scripts/ci/desktop-release.sh npm-publish "$RUNNER_TEMP/npm" "$VERSION" --publish diff --git a/apps/desktop/package.json b/apps/desktop/package.json index 68ab85540..5cda3e124 100644 --- a/apps/desktop/package.json +++ b/apps/desktop/package.json @@ -14,6 +14,7 @@ }, "dependencies": { "@linger/local-installation": "workspace:*", + "@linger/service-kit": "workspace:*", "@linger/wire-contracts": "workspace:*", "yazl": "3.3.1", "zod": "4.4.3" diff --git a/apps/desktop/src/main/index.ts b/apps/desktop/src/main/index.ts index b1220d566..b36edda4d 100644 --- a/apps/desktop/src/main/index.ts +++ b/apps/desktop/src/main/index.ts @@ -217,6 +217,7 @@ async function main(): Promise { ? updateInstallFor(process.platform, process.execPath, existsSync) : undefined, downloadDirectory: path.join(userData, "updates"), + home, announce: (state) => { if (!Notification.isSupported()) return; const notice = new Notification(updateNotice(state)); diff --git a/apps/desktop/src/main/test-support.ts b/apps/desktop/src/main/test-support.ts index 9c0aac4ff..ca1c15234 100644 --- a/apps/desktop/src/main/test-support.ts +++ b/apps/desktop/src/main/test-support.ts @@ -1,6 +1,7 @@ import { mkdtempSync } from "node:fs"; import os from "node:os"; import path from "node:path"; +import { gzipSync } from "node:zlib"; import { ConfigStore, type DesktopConfig } from "./config.js"; import type { Logger } from "./log.js"; @@ -35,3 +36,38 @@ export const stubModule = (name: string, dependsOn: string[] = []): Module => ({ start: async () => {}, stop: async () => {}, }); + +// One tar member as `npm pack` writes it: a ustar header, the body and the +// padding to the next 512-byte block. `type` "0" is a regular file. +export const tarEntry = (name: string, body: Buffer, type = "0"): Buffer => { + const header = Buffer.alloc(512); + header.write(name, 0, 100, "utf8"); + header.write("0000644\0", 100); + header.write("0000000\0", 108); + header.write("0000000\0", 116); + header.write(`${body.length.toString(8).padStart(11, "0")}\0`, 124); + header.write("00000000000\0", 136); + header.write(" ", 148); + header.write(type, 156); + header.write("ustar\0", 257); + header.write("00", 263); + let sum = 0; + for (const byte of header) sum += byte; + header.write(`${sum.toString(8).padStart(6, "0")}\0 `, 148); + return Buffer.concat([ + header, + body, + Buffer.alloc((512 - (body.length % 512)) % 512), + ]); +}; + +/** A gzipped tarball of `members` (name, body[, type]) and its end blocks. */ +export const npmTarball = ( + members: readonly (readonly [string, Buffer, string?])[], +): Buffer => + gzipSync( + Buffer.concat([ + ...members.map(([name, body, type]) => tarEntry(name, body, type)), + Buffer.alloc(1024), + ]), + ); diff --git a/apps/desktop/src/main/update-manifest.test.ts b/apps/desktop/src/main/update-manifest.test.ts index e81d4e988..1a9f109d3 100644 --- a/apps/desktop/src/main/update-manifest.test.ts +++ b/apps/desktop/src/main/update-manifest.test.ts @@ -80,6 +80,40 @@ describe("update manifest", () => { ); }); + it("reads the npm parts of a package", () => { + const { privateKey, publicPem } = keys(); + const parts = [ + { sha256: "b".repeat(64), size: 6 }, + { sha256: "c".repeat(64), size: 4 }, + ]; + const withParts = structuredClone(manifest); + Object.assign(withParts.packages["darwin-arm64"], { parts }); + const { bytes, signature } = signed(withParts, privateKey); + expect( + packageFor(verifyManifest(bytes, signature, publicPem), "darwin", "arm64") + ?.parts, + ).toEqual(parts); + + const bad = structuredClone(manifest); + Object.assign(bad.packages["darwin-arm64"], { + parts: [{ sha256: "nope", size: 6 }], + }); + const refused = signed(bad, privateKey); + expect(() => + verifyManifest(refused.bytes, refused.signature, publicPem), + ).toThrow("Update manifest is invalid"); + }); + + // Apps read manifests of releases newer than themselves. + it("ignores keys it does not know", () => { + const { privateKey, publicPem } = keys(); + const newer = structuredClone(manifest); + Object.assign(newer, { channel: "stable" }); + Object.assign(newer.packages["darwin-arm64"], { mirrors: ["x"] }); + const { bytes, signature } = signed(newer, privateKey); + expect(verifyManifest(bytes, signature, publicPem)).toEqual(manifest); + }); + it("compares versions numerically", () => { expect(compareVersions("0.16.10", "0.16.9")).toBeGreaterThan(0); expect(compareVersions("0.16.0", "0.16.0")).toBe(0); diff --git a/apps/desktop/src/main/update-manifest.ts b/apps/desktop/src/main/update-manifest.ts index d4cb16ac0..9b69d1593 100644 --- a/apps/desktop/src/main/update-manifest.ts +++ b/apps/desktop/src/main/update-manifest.ts @@ -20,14 +20,35 @@ MCowBQYDK2VwAyEA985QJ8UAvzK0ZLPW6YPaIkiCS2A+qPbEaZvi0aAccR4= const VERSION = /^\d+\.\d+\.\d+(-[0-9A-Za-z][0-9A-Za-z.-]*)?$/; +const SHA256 = /^[0-9a-f]{64}$/; + +// Not strict: an app ignores keys a newer release adds, as apps from before +// `parts` ignore that one. const updatePackage = z.object({ // A bare file name: the package always comes from the manifest's release. file: z.string().regex(/^[A-Za-z0-9._-]+$/), - sha256: z.string().regex(/^[0-9a-f]{64}$/), + sha256: z.string().regex(SHA256), size: z.number().int().positive(), + // The package cut into consecutive pieces, each its own npm package + // (update-npm.ts), so the China mirror can serve it. The pieces join into + // the whole file above. + parts: z + .array( + z.object({ + sha256: z.string().regex(SHA256), + size: z.number().int().positive(), + }), + ) + .min(1) + .optional(), }); export type UpdatePackage = z.infer; +/** Whether `value` is a version a manifest may name. */ +export function isReleaseVersion(value: unknown): value is string { + return typeof value === "string" && VERSION.test(value); +} + const updateManifest = z.object({ schema: z.literal(2), version: z.string().regex(VERSION), diff --git a/apps/desktop/src/main/update-npm.test.ts b/apps/desktop/src/main/update-npm.test.ts new file mode 100644 index 000000000..aea7f656e --- /dev/null +++ b/apps/desktop/src/main/update-npm.test.ts @@ -0,0 +1,154 @@ +import { gzipSync } from "node:zlib"; + +import { describe, expect, it } from "vitest"; + +import { npmTarball, tarEntry } from "./test-support.js"; +import { + NPM_PART_MEMBER, + NpmTarError, + distTagsUrl, + extractNpmTarball, + npmTarballUrl, + partNpmPackage, +} from "./update-npm.js"; + +const PACKAGE_JSON = Buffer.from('{"name":"x","version":"1.0.0"}'); + +async function* chunked(bytes: Buffer, size = 7): AsyncGenerator { + for (let i = 0; i < bytes.length; i += size) + yield bytes.subarray(i, i + size); +} + +const extract = async ( + tarball: Buffer, + wanted: Record = { [NPM_PART_MEMBER]: 4096 }, + chunk = 7, +) => { + const files: Record = {}; + await extractNpmTarball(chunked(tarball, chunk), wanted, (member, chunk) => { + (files[member] ??= []).push(Buffer.from(chunk)); + }); + return Object.fromEntries( + Object.entries(files).map(([name, chunks]) => [ + name, + Buffer.concat(chunks).toString(), + ]), + ); +}; + +describe("npm update packages", () => { + it("names the packages and where each registry serves them", () => { + expect(partNpmPackage("darwin-arm64", 0)).toBe( + "@simplexai/linger-desktop-darwin-arm64-part1", + ); + const name = "@simplexai/linger-desktop-update"; + expect( + distTagsUrl("https://registry.npmmirror.com/", name).toString(), + ).toBe( + "https://registry.npmmirror.com/-/package/@simplexai/linger-desktop-update/dist-tags", + ); + expect( + npmTarballUrl( + "https://registry.npmmirror.com/", + name, + "1.2.3", + ).toString(), + ).toBe( + "https://cdn.npmmirror.com/packages/@simplexai/linger-desktop-update/1.2.3/linger-desktop-update-1.2.3.tgz", + ); + expect( + npmTarballUrl("https://registry.npmjs.org/", name, "1.2.3").toString(), + ).toBe( + "https://registry.npmjs.org/@simplexai/linger-desktop-update/-/linger-desktop-update-1.2.3.tgz", + ); + }); +}); + +describe("extractNpmTarball", () => { + it("hands over the wanted member and reads past package.json", async () => { + const body = Buffer.alloc(1300, "p"); + const files = await extract( + npmTarball([ + ["package/package.json", PACKAGE_JSON], + [NPM_PART_MEMBER, body], + ]), + ); + expect(files).toEqual({ [NPM_PART_MEMBER]: body.toString() }); + }); + + // Bytes after the end blocks, however many they inflate to, are not read. + it("stops at the end of the archive", async () => { + const tar = Buffer.concat([ + tarEntry(NPM_PART_MEMBER, Buffer.from("part")), + Buffer.alloc(1024), + tarEntry("package/evil", Buffer.from("never read")), + Buffer.alloc(64 * 1024 * 1024, 1), + ]); + expect(await extract(gzipSync(tar), undefined, 64 * 1024)).toEqual({ + [NPM_PART_MEMBER]: "part", + }); + }); + + it.each([ + ["another file", [["package/other", Buffer.from("x")]]], + ["a path outside package/", [["part", Buffer.from("x")]]], + ["a traversal", [["package/../part", Buffer.from("x")]]], + ["a directory", [[NPM_PART_MEMBER, Buffer.alloc(0), "5"]]], + ["a symlink", [[NPM_PART_MEMBER, Buffer.alloc(0), "2"]]], + ["a pax header", [[NPM_PART_MEMBER, Buffer.from("x"), "x"]]], + ] as const)("rejects %s", async (_label, members) => { + await expect( + extract(npmTarball([...members, [NPM_PART_MEMBER, Buffer.from("p")]])), + ).rejects.toThrow(NpmTarError); + }); + + it("rejects a member over its size before reading it", async () => { + const written: Buffer[] = []; + await expect( + extractNpmTarball( + chunked(npmTarball([[NPM_PART_MEMBER, Buffer.alloc(5000)]])), + { [NPM_PART_MEMBER]: 4096 }, + (_member, chunk) => { + written.push(chunk); + }, + ), + ).rejects.toThrow("larger than expected"); + expect(written).toEqual([]); + }); + + it("rejects an oversized package.json", async () => { + await expect( + extract( + npmTarball([ + ["package/package.json", Buffer.alloc(70 * 1024)], + [NPM_PART_MEMBER, Buffer.from("p")], + ]), + ), + ).rejects.toThrow(NpmTarError); + }); + + it("rejects a repeated, missing or damaged member", async () => { + await expect( + extract( + npmTarball([ + [NPM_PART_MEMBER, Buffer.from("a")], + [NPM_PART_MEMBER, Buffer.from("b")], + ]), + ), + ).rejects.toThrow("repeats"); + await expect( + extract(npmTarball([["package/package.json", PACKAGE_JSON]])), + ).rejects.toThrow("missing"); + const entry = tarEntry(NPM_PART_MEMBER, Buffer.from("p")); + entry[0] = "q".charCodeAt(0); + await expect( + extract(gzipSync(Buffer.concat([entry, Buffer.alloc(1024)]))), + ).rejects.toThrow("malformed"); + const cut = tarEntry(NPM_PART_MEMBER, Buffer.alloc(600)).subarray(0, 700); + await expect(extract(gzipSync(cut))).rejects.toThrow("ends early"); + }); + + it("rejects bytes that are not gzip", async () => { + await expect(extract(Buffer.from("not gzip at all"))).rejects.toThrow(); + }); +}); diff --git a/apps/desktop/src/main/update-npm.ts b/apps/desktop/src/main/update-npm.ts new file mode 100644 index 000000000..eac9b496d --- /dev/null +++ b/apps/desktop/src/main/update-npm.ts @@ -0,0 +1,172 @@ +import { Readable } from "node:stream"; +import { createGunzip } from "node:zlib"; + +import { + npmMirrorRegistry, + npmRegistry, +} from "@linger/local-installation/npm-registry"; + +// The update feed as npm packages, which the China mirror (npmmirror) +// serves where GitHub is slow or unreachable. Each release version V +// publishes UPDATE_NPM_PACKAGE@V with the signed manifest.json and +// manifest.json.sig, and every package that the manifest splits into `parts` +// as one npm package per part, `partNpmPackage(target, n)`@V, holding the +// single file `part`. scripts/ci/desktop-release.sh (npm-pack, npm-publish) +// writes them; nothing here is trusted before the manifest signature and the +// sha256 of each part and of the whole package match. +export const UPDATE_NPM_PACKAGE = "@simplexai/linger-desktop-update"; +export const NPM_MANIFEST_MEMBER = "package/manifest.json"; +export const NPM_SIGNATURE_MEMBER = "package/manifest.json.sig"; +export const NPM_PART_MEMBER = "package/part"; +// npm pack always adds it; it is read past, never used. +const PACKAGE_JSON_MEMBER = "package/package.json"; +const PACKAGE_JSON_MAX = 64 * 1024; + +export const NPM_REGISTRIES = [npmMirrorRegistry, npmRegistry] as const; + +/** The npm package of part `index` (from 0) of a target's update package. */ +export function partNpmPackage(target: string, index: number): string { + return `@simplexai/linger-desktop-${target}-part${index + 1}`; +} + +export function distTagsUrl(registry: string, name: string): URL { + return new URL(`-/package/${name}/dist-tags`, registry); +} + +// The mirror's registry redirects tarballs to its CDN; asking the CDN saves +// the round trip. +export function npmTarballUrl( + registry: string, + name: string, + version: string, +): URL { + const basename = name.slice(name.lastIndexOf("/") + 1); + return registry === npmMirrorRegistry + ? new URL( + `https://cdn.npmmirror.com/packages/${name}/${version}/${basename}-${version}.tgz`, + ) + : new URL(`${name}/-/${basename}-${version}.tgz`, registry); +} + +export class NpmTarError extends Error {} + +const BLOCK = 512; + +// Pulls exact byte counts out of a stream of chunks. +class ByteReader { + readonly #chunks: AsyncIterator; + #pending: Buffer = Buffer.alloc(0); + + constructor(chunks: AsyncIterable) { + this.#chunks = chunks[Symbol.asyncIterator](); + } + + async #fill(): Promise { + const next = await this.#chunks.next(); + if (next.done) return false; + this.#pending = + this.#pending.length === 0 + ? next.value + : Buffer.concat([this.#pending, next.value]); + return true; + } + + async exactly(size: number): Promise { + while (this.#pending.length < size) + if (!(await this.#fill())) + throw new NpmTarError("Update package ends early"); + const bytes = this.#pending.subarray(0, size); + this.#pending = this.#pending.subarray(size); + return bytes; + } + + // Yields `size` bytes as they arrive, without holding them all. + async *stream(size: number): AsyncGenerator { + let left = size; + while (left > 0) { + if (this.#pending.length === 0 && !(await this.#fill())) + throw new NpmTarError("Update package ends early"); + const take = Math.min(left, this.#pending.length); + yield this.#pending.subarray(0, take); + this.#pending = this.#pending.subarray(take); + left -= take; + } + } +} + +const text = (header: Buffer, start: number, length: number): string => { + const field = header.subarray(start, start + length); + const end = field.indexOf(0); + return field.subarray(0, end < 0 ? length : end).toString("utf8"); +}; + +const octal = (header: Buffer, start: number, length: number): number => { + const value = text(header, start, length).trim(); + if (!/^[0-7]+$/.test(value)) + throw new NpmTarError("Update package has a malformed entry"); + return parseInt(value, 8); +}; + +// The checksum counts its own field as eight spaces. +const checksumMatches = (header: Buffer): boolean => { + let sum = 0; + for (let i = 0; i < BLOCK; i++) + sum += i >= 148 && i < 156 ? 0x20 : header[i]!; + return sum === octal(header, 148, 8); +}; + +/** + * Reads a gzipped npm tarball and hands each wanted member's bytes to + * `write`. `wanted` maps a member name (such as `package/part`) to its largest + * accepted size. Only regular files are accepted: each wanted member exactly + * once, and npm's own package.json, which is skipped. Anything else (another + * name, a directory, a link, a pax header, a member over its size) rejects the + * whole tarball. Reading stops at the end-of-archive block. + */ +export async function extractNpmTarball( + gzipped: AsyncIterable, + wanted: Readonly>, + write: (member: string, chunk: Buffer) => void | Promise, +): Promise { + const seen = new Set(); + const source = Readable.from(gzipped); + const gunzip = createGunzip(); + source.on("error", (error) => gunzip.destroy(error)); + source.pipe(gunzip); + // Leaving at the end-of-archive block tears both streams down, so bytes + // after it are neither inflated nor downloaded. + try { + const reader = new ByteReader(gunzip); + for (;;) { + const header = await reader.exactly(BLOCK); + if (header.every((byte) => byte === 0)) break; + if (!checksumMatches(header)) + throw new NpmTarError("Update package has a malformed entry"); + const prefix = text(header, 345, 155); + const name = prefix + ? `${prefix}/${text(header, 0, 100)}` + : text(header, 0, 100); + const type = String.fromCharCode(header[156]!); + const size = octal(header, 124, 12); + const max = + name === PACKAGE_JSON_MEMBER ? PACKAGE_JSON_MAX : wanted[name]; + if ((type !== "0" && type !== "\0") || max === undefined) + throw new NpmTarError("Update package has an unexpected entry"); + if (size > max) + throw new NpmTarError("Update package entry is larger than expected"); + if (seen.has(name)) + throw new NpmTarError("Update package repeats an entry"); + seen.add(name); + for await (const chunk of reader.stream(size)) + if (name !== PACKAGE_JSON_MEMBER) await write(name, chunk); + const padding = (BLOCK - (size % BLOCK)) % BLOCK; + if (padding) await reader.exactly(padding); + } + } finally { + source.destroy(); + gunzip.destroy(); + } + for (const name of Object.keys(wanted)) + if (!seen.has(name)) + throw new NpmTarError("Update package is missing an entry"); +} diff --git a/apps/desktop/src/main/updater.test.ts b/apps/desktop/src/main/updater.test.ts index e51a70d99..7b98f7645 100644 --- a/apps/desktop/src/main/updater.test.ts +++ b/apps/desktop/src/main/updater.test.ts @@ -3,11 +3,22 @@ import { EventEmitter } from "node:events"; import { mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs"; import path from "node:path"; -import { describe, expect, it } from "vitest"; +import { beforeEach, describe, expect, it, vi } from "vitest"; import type { ComputerDaemonModule } from "./computer-daemon.js"; import type { DesktopConfig } from "./config.js"; -import { stubModule, tempDirectory, testHost } from "./test-support.js"; +import { + npmTarball, + stubModule, + tempDirectory, + testHost, +} from "./test-support.js"; +import { + UPDATE_NPM_PACKAGE, + distTagsUrl, + npmTarballUrl, + partNpmPackage, +} from "./update-npm.js"; import { type SpawnSetup, type UpdateState, @@ -18,7 +29,20 @@ import { } from "./updater.js"; const RELEASES = "https://github.com/simplexai-labs/linger/releases"; +const MIRROR = "https://registry.npmmirror.com/"; +const NPMJS = "https://registry.npmjs.org/"; const PACKAGE = Buffer.from("new app archive"); +const sha256 = (bytes: Buffer) => + createHash("sha256").update(bytes).digest("hex"); +const PACKAGE_JSON: [string, Buffer] = [ + "package/package.json", + Buffer.from('{"name":"x"}'), +]; + +beforeEach(() => { + vi.stubEnv("SIMPLEX_REGION", ""); + return () => vi.unstubAllEnvs(); +}); const release = ( version: string, @@ -27,32 +51,73 @@ const release = ( target = "darwin-arm64", file = `Linger-${version}-macos-arm64.app.tar.gz`, feed = RELEASES, + // Splits the package into parts of this size, as the release does. + partSize = 0, + // Registries that hold the npm packages of the release. + npm = [] as readonly string[], + // The version npm has, when it lags the release. + npmVersion = version, } = {}, ) => { const { privateKey, publicKey } = generateKeyPairSync("ed25519"); - const manifest = Buffer.from( - JSON.stringify({ - schema: 2, - version, - packages: { - [target]: { - file, - sha256: createHash("sha256").update(PACKAGE).digest("hex"), - size: PACKAGE.length, + const pieces: Buffer[] = []; + for (let i = 0; partSize && i < PACKAGE.length; i += partSize) + pieces.push(PACKAGE.subarray(i, i + partSize)); + const manifestOf = (version: string) => + Buffer.from( + JSON.stringify({ + schema: 2, + version, + packages: { + [target]: { + file, + sha256: sha256(PACKAGE), + size: PACKAGE.length, + ...(partSize + ? { + parts: pieces.map((piece) => ({ + sha256: sha256(piece), + size: piece.length, + })), + } + : {}), + }, }, - }, - }), - ); + }), + ); + const signatureOf = (manifest: Buffer) => + Buffer.from(sign(null, manifest, privateKey).toString("base64")); + const manifest = manifestOf(version); + const signature = signatureOf(manifest); + const npmManifest = manifestOf(npmVersion); const files: Record = { [`${feed}/latest/download/manifest.json`]: manifest, - [`${feed}/latest/download/manifest.json.sig`]: Buffer.from( - sign(null, manifest, privateKey).toString("base64"), - ), + [`${feed}/latest/download/manifest.json.sig`]: signature, [`${feed}/download/desktop-v${version}/${file}`]: body, }; + for (const registry of npm) { + files[distTagsUrl(registry, UPDATE_NPM_PACKAGE).toString()] = Buffer.from( + JSON.stringify({ latest: npmVersion }), + ); + files[npmTarballUrl(registry, UPDATE_NPM_PACKAGE, npmVersion).toString()] = + npmTarball([ + PACKAGE_JSON, + ["package/manifest.json", npmManifest], + ["package/manifest.json.sig", signatureOf(npmManifest)], + ]); + pieces.forEach((piece, index) => { + files[partUrl(registry, target, index, npmVersion)] = npmTarball([ + PACKAGE_JSON, + ["package/part", piece], + ]); + }); + } const requested: string[] = []; + // Hosts whose connections fail. + const unreachable = new Set(); const fetch = (async (url: URL) => { requested.push(url.toString()); + if (unreachable.has(url.host)) throw new TypeError("fetch failed"); const content = files[url.toString()]; return content ? new Response(new Uint8Array(content)) @@ -61,17 +126,44 @@ const release = ( return { fetch, files, + manifest, + signature, requested, + unreachable, publicKey: publicKey.export({ type: "spki", format: "pem" }).toString(), }; }; +const partUrl = ( + registry: string, + target: string, + index: number, + version: string, +) => npmTarballUrl(registry, partNpmPackage(target, index), version).toString(); +const manifestUrl = (registry: string, version: string) => + npmTarballUrl(registry, UPDATE_NPM_PACKAGE, version).toString(); +const tagsUrl = (registry: string) => + distTagsUrl(registry, UPDATE_NPM_PACKAGE).toString(); + const setup = async ( feed: ReturnType, options: Partial = {}, config: Partial = {}, + region: "cn" | "global" = "global", ) => { const root = tempDirectory(); + // The home's download region, resolved earlier (region.ts). + const home = path.join(root, "home"); + mkdirSync(path.join(home, "runtime"), { recursive: true }); + writeFileSync( + path.join(home, "runtime", "region.json"), + JSON.stringify({ + schema: "simplex.region/v1", + region, + resolved_at: new Date().toISOString(), + method: "probe", + }), + ); const applications = path.join(root, "Applications"); const bundle = path.join(applications, "Linger.app"); mkdirSync(bundle, { recursive: true }); @@ -92,6 +184,7 @@ const setup = async ( arch: "arm64", install: { kind: "app-bundle", appBundle: bundle }, downloadDirectory: downloads, + home, restart: async () => { restarts += 1; }, @@ -125,6 +218,11 @@ const setup = async ( bundle, applications, downloads, + home, + region: () => + JSON.parse( + readFileSync(path.join(home, "runtime", "region.json"), "utf8"), + ) as { region: string; method: string }, setRunning, opened, announced, @@ -179,6 +277,7 @@ describe("UpdaterModule", () => { arch: "arm64", install: { kind: "app-bundle", appBundle: bundle }, downloadDirectory: tempDirectory(), + home: tempDirectory(), restart: async () => {}, quit: async () => {}, openExternal: async () => {}, @@ -282,6 +381,219 @@ describe("UpdaterModule", () => { }); }); +describe("UpdaterModule with the npm packages", () => { + const VERSION = "0.17.0"; + const TARGET = "darwin-arm64"; + const ASSET = `${RELEASES}/download/desktop-v${VERSION}/Linger-${VERSION}-macos-arm64.app.tar.gz`; + const parts = (registry: string) => + [0, 1, 2].map((index) => partUrl(registry, TARGET, index, VERSION)); + const published = (npm: readonly string[] = [MIRROR, NPMJS]) => + release(VERSION, { partSize: 6, npm }); + + it("in cn takes the manifest and the parts from the China mirror", async () => { + const feed = published(); + const { updater, bundle, downloads, restarts, region } = await setup( + feed, + {}, + {}, + "cn", + ); + await updater.check(); + expect(updater.state()).toEqual({ kind: "ready", version: VERSION }); + expect(feed.requested).toEqual([ + tagsUrl(MIRROR), + manifestUrl(MIRROR, VERSION), + ...parts(MIRROR), + ]); + expect(feed.requested[1]).toMatch(/^https:\/\/cdn\.npmmirror\.com\//); + expect(readdirSync(downloads)).toEqual([]); + await updater.install(); + await waitFor(() => restarts() === 1); + // The joined parts are the package the manifest names. + expect(readFileSync(path.join(bundle, "version"))).toEqual(PACKAGE); + expect(region()).toMatchObject({ region: "cn", method: "probe" }); + }); + + it("in cn takes a tampered part from npm instead and prefers global after", async () => { + const feed = published(); + feed.files[parts(MIRROR)[1]!] = npmTarball([ + ["package/part", Buffer.from("XXXXXX")], + ]); + const { updater, bundle, restarts, region } = await setup( + feed, + {}, + {}, + "cn", + ); + await updater.check(); + expect(updater.state()).toEqual({ kind: "ready", version: VERSION }); + expect(feed.requested.slice(2)).toEqual([ + parts(MIRROR)[0], + parts(MIRROR)[1], + parts(NPMJS)[1], + parts(MIRROR)[2], + ]); + await updater.install(); + await waitFor(() => restarts() === 1); + expect(readFileSync(path.join(bundle, "version"))).toEqual(PACKAGE); + expect(region()).toMatchObject({ region: "global", method: "fallback" }); + }); + + it("in cn takes the release asset when a part is on no registry", async () => { + const feed = published([MIRROR]); + delete feed.files[parts(MIRROR)[2]!]; + const { updater, downloads, region } = await setup(feed, {}, {}, "cn"); + await updater.check(); + expect(updater.state()).toEqual({ kind: "ready", version: VERSION }); + expect(feed.requested.slice(-3)).toEqual([ + parts(MIRROR)[2], + parts(NPMJS)[2], + ASSET, + ]); + expect(readdirSync(downloads)).toEqual([]); + // A part the mirror has not synced yet says nothing about the network. + expect(region()).toMatchObject({ region: "cn", method: "probe" }); + }); + + it("in cn refuses a manifest whose package was tampered with", async () => { + const feed = published([MIRROR]); + const tampered = Buffer.from( + feed.manifest.toString().replace(sha256(PACKAGE), "0".repeat(64)), + ); + feed.files[manifestUrl(MIRROR, VERSION)] = npmTarball([ + ["package/manifest.json", tampered], + ["package/manifest.json.sig", feed.signature], + ]); + delete feed.files[`${RELEASES}/latest/download/manifest.json`]; + const { updater } = await setup(feed, {}, {}, "cn"); + await updater.check(); + expect(updater.state()).toEqual({ + kind: "failed", + error: "Update signature does not match", + }); + expect(feed.requested.some((url) => url.includes("-part"))).toBe(false); + }); + + it("in cn checks a manifest from npm against the built-in key", async () => { + // Signed by the release key, so only UPDATE_PUBLIC_KEY decides. + const feed = published([MIRROR]); + const { updater } = await setup(feed, { publicKey: undefined }, {}, "cn"); + await updater.check(); + expect(updater.state()).toEqual({ + kind: "failed", + error: "Update signature does not match", + }); + }); + + it("in cn falls back to the release feed when npm is unreachable", async () => { + const feed = published(); + feed.unreachable.add("registry.npmmirror.com"); + feed.unreachable.add("cdn.npmmirror.com"); + feed.unreachable.add("registry.npmjs.org"); + const { updater, region } = await setup(feed, {}, {}, "cn"); + await updater.check(); + expect(updater.state()).toEqual({ kind: "ready", version: VERSION }); + expect(feed.requested).toEqual([ + tagsUrl(MIRROR), + tagsUrl(NPMJS), + `${RELEASES}/latest/download/manifest.json`, + `${RELEASES}/latest/download/manifest.json.sig`, + // A manifest only the release served: its package comes from there. + ASSET, + ]); + expect(region()).toMatchObject({ region: "global", method: "fallback" }); + }); + + it("in cn takes a version npm lacks from the release", async () => { + // npm still has the running version: its publish failed or was skipped. + const feed = release(VERSION, { + partSize: 6, + npm: [MIRROR, NPMJS], + npmVersion: "0.16.7", + }); + const { updater, bundle, restarts, region } = await setup( + feed, + {}, + {}, + "cn", + ); + await updater.check(); + expect(updater.state()).toEqual({ kind: "ready", version: VERSION }); + expect(feed.requested).toEqual([ + tagsUrl(MIRROR), + manifestUrl(MIRROR, "0.16.7"), + tagsUrl(NPMJS), + manifestUrl(NPMJS, "0.16.7"), + `${RELEASES}/latest/download/manifest.json`, + `${RELEASES}/latest/download/manifest.json.sig`, + ASSET, + ]); + await updater.install(); + await waitFor(() => restarts() === 1); + expect(readFileSync(path.join(bundle, "version"))).toEqual(PACKAGE); + expect(region()).toMatchObject({ region: "cn", method: "probe" }); + }); + + it("in cn stays idle when npm is current and GitHub cannot be reached", async () => { + const feed = release("0.16.7", { partSize: 6, npm: [MIRROR, NPMJS] }); + feed.unreachable.add("github.com"); + const { updater } = await setup(feed, {}, {}, "cn"); + await updater.check(); + expect(updater.state()).toEqual({ kind: "idle" }); + }); + + it("does not offer a prerelease npm tags latest", async () => { + const feed = release("0.17.0-rc.1", { partSize: 6, npm: [MIRROR] }); + delete feed.files[`${RELEASES}/latest/download/manifest.json`]; + const { updater } = await setup(feed, {}, {}, "cn"); + await updater.check(); + expect(updater.state()).toEqual({ kind: "idle" }); + expect(feed.requested).toContain(tagsUrl(MIRROR)); + expect(feed.requested).not.toContain(manifestUrl(MIRROR, "0.17.0-rc.1")); + }); + + it("in global keeps to the release while it answers", async () => { + const feed = published(); + const { updater } = await setup(feed); + await updater.check(); + expect(updater.state()).toEqual({ kind: "ready", version: VERSION }); + expect(feed.requested).toEqual([ + `${RELEASES}/latest/download/manifest.json`, + `${RELEASES}/latest/download/manifest.json.sig`, + ASSET, + ]); + }); + + it("in global falls back to npm when the release cannot be reached", async () => { + const feed = published(); + feed.unreachable.add("github.com"); + const { updater, bundle, restarts, region } = await setup(feed); + await updater.check(); + expect(updater.state()).toEqual({ kind: "ready", version: VERSION }); + expect(feed.requested).toEqual([ + `${RELEASES}/latest/download/manifest.json`, + `${RELEASES}/latest/download/manifest.json.sig`, + tagsUrl(NPMJS), + manifestUrl(NPMJS, VERSION), + ASSET, + ...parts(NPMJS), + ]); + await updater.install(); + await waitFor(() => restarts() === 1); + expect(readFileSync(path.join(bundle, "version"))).toEqual(PACKAGE); + expect(region()).toMatchObject({ region: "global", method: "probe" }); + }); + + it("asks no npm registry for a feed of its own", async () => { + const own = "https://releases.example.com/linger"; + const feed = release(VERSION, { feed: own, partSize: 6, npm: [MIRROR] }); + const { updater } = await setup(feed, {}, { updateFeedUrl: own }, "cn"); + await updater.check(); + expect(updater.state()).toEqual({ kind: "ready", version: VERSION }); + expect(feed.requested.every((url) => url.startsWith(own))).toBe(true); + }); +}); + describe("UpdaterModule with the Windows setup program", () => { const SETUP = "Linger-0.17.0-windows-x64-setup.exe"; const windows = async (spawn: SpawnSetup) => { diff --git a/apps/desktop/src/main/updater.ts b/apps/desktop/src/main/updater.ts index 0a056acfa..52bca01d2 100644 --- a/apps/desktop/src/main/updater.ts +++ b/apps/desktop/src/main/updater.ts @@ -1,12 +1,19 @@ import { execFile, spawn as spawnProcess } from "node:child_process"; import { createHash } from "node:crypto"; import { createReadStream, createWriteStream } from "node:fs"; -import { mkdir, readdir, rename, rm, stat } from "node:fs/promises"; +import { mkdir, open, readdir, rename, rm, stat } from "node:fs/promises"; import path from "node:path"; import { Readable } from "node:stream"; import { pipeline } from "node:stream/promises"; import { promisify } from "node:util"; +import { + recordRegionFallback, + resolveRegion, + type Region, +} from "@linger/local-installation/region"; +import { orderByRegion, sourceRegion } from "@linger/service-kit"; + import type { ComputerDaemonModule } from "./computer-daemon.js"; import type { Module, ModuleContext } from "./modules.js"; import { @@ -15,12 +22,25 @@ import { UPDATE_PUBLIC_KEY, compareVersions, feedUrls, + isReleaseVersion, packageFor, packageUrl, releasePageUrl, verifyManifest, + type UpdateManifest, type UpdatePackage, } from "./update-manifest.js"; +import { + NPM_MANIFEST_MEMBER, + NPM_PART_MEMBER, + NPM_REGISTRIES, + NPM_SIGNATURE_MEMBER, + UPDATE_NPM_PACKAGE, + distTagsUrl, + extractNpmTarball, + npmTarballUrl, + partNpmPackage, +} from "./update-npm.js"; export type UpdateState = | { readonly kind: "idle" } @@ -62,6 +82,9 @@ export interface UpdaterOptions { readonly install: UpdateInstall | undefined; // Downloads live here until they are installed. readonly downloadDirectory: string; + // The Linger home whose download region (region.ts) orders the sources + // of the default feed. + readonly home: string; // Stops modules, including the daemon, and relaunches the app from // the app bundle, which by then holds the new version and its daemon. readonly restart: () => Promise; @@ -99,6 +122,8 @@ const CHECK_DELAY_MS = 60_000; const CHECK_INTERVAL_MS = 60 * 60_000; // A dropped connection should not cost a whole check interval. const RETRY_DELAY_MS = 5 * 60_000; +const MANIFEST_MAX = 1024 * 1024; +const SIGNATURE_MAX = 4096; const STAGED_PREFIX = ".Linger-update-"; const OLD_PREFIX = ".Linger-old-"; @@ -110,10 +135,29 @@ const untar = async (archive: string, directory: string): Promise => { }; export class UpdateError extends Error {} +// The source does not have the file (yet: the mirror syncs a new version +// minutes after it is published). Unlike other failures it says nothing +// about the region's network. +class UpdateNotFound extends UpdateError {} + +// Where one check takes its files from. +interface Route { + readonly region: Region; + // The npm packages mirror the default feed only; a feed of its own is the + // only source. + readonly npm: boolean; + // A source in `region` failed for a reason other than not having the file. + regionFailed: boolean; + // The manifest came from npm, so its parts are there too; in cn a manifest + // only the release had is downloaded from the release. + fromNpm: boolean; +} // Self-update without Squirrel: a signed manifest in the newest GitHub // Release (or a feed with the same layout), a sha256-checked package from -// that same release, and an install that waits for running tasks. Sign-in and +// that same release, and an install that waits for running tasks. The same +// manifest and the package in parts also come as npm packages, which the +// China mirror serves to homes in the cn region (update-npm.ts). Sign-in and // the computer identity live outside the installed app, so they carry over. export class UpdaterModule implements Module { readonly name = "Updater"; @@ -214,28 +258,18 @@ export class UpdaterModule implements Module { async #check(): Promise { const ctx = this.#ctx!; const install = this.#options.install!; - const feed = ctx.config.get().updateFeedUrl ?? DEFAULT_FEED_URL; + const configured = ctx.config.get().updateFeedUrl; + const feed = configured ?? DEFAULT_FEED_URL; this.#setState({ kind: "checking" }); try { if (new URL(feed).protocol !== "https:") throw new UpdateError("The update feed must use https"); - const urls = feedUrls(feed); - const [bytes, signature] = await Promise.all([ - this.#get(urls.manifest), - this.#get(urls.signature), - ]); - // The newest release was published without a manifest: nothing to - // offer until one is. - if (!bytes || !signature) { - ctx.log.info("update.no_manifest", { feed }); + const route = await this.#route(configured === undefined); + const manifest = await this.#manifest(feed, route); + if (!manifest) { this.#setState({ kind: "idle" }); return; } - const manifest = verifyManifest( - bytes, - signature.toString("utf8"), - this.#options.publicKey ?? UPDATE_PUBLIC_KEY, - ); const pkg = packageFor( manifest, this.#options.platform, @@ -260,7 +294,7 @@ export class UpdaterModule implements Module { } this.#setState({ kind: "downloading", version }); const file = path.join(this.#options.downloadDirectory, pkg.file); - await this.#download(packageUrl(feed, version, pkg.file), file, pkg); + await this.#downloadPackage(feed, version, pkg, file, route); this.#staged = install.kind === "app-bundle" ? { path: await this.#extract(file, version), pkg } @@ -273,6 +307,337 @@ export class UpdaterModule implements Module { } } + async #route(npm: boolean): Promise { + let region: Region = "global"; + if (npm) { + try { + region = await resolveRegion(this.#options.home, { + fetcher: this.#options.fetch ?? fetch, + }); + } catch (error) { + this.#ctx!.log.warn("update.region_failed", { error: String(error) }); + } + } + return { region, npm, regionFailed: false, fromNpm: false }; + } + + #failed(route: Route, source: URL, error: unknown): void { + this.#ctx!.log.warn("update.source_failed", { + source: source.host, + error: error instanceof Error ? error.message : String(error), + }); + if ( + !(error instanceof UpdateNotFound) && + sourceRegion(source) === route.region + ) + route.regionFailed = true; + } + + // The other region served what the home's region could not: downloads + // from this home prefer it from now on. + async #served(route: Route, source: URL): Promise { + const served = sourceRegion(source); + if (!route.regionFailed || served === route.region) return; + route.regionFailed = false; + this.#ctx!.log.info("update.region_fallback", { region: served }); + await recordRegionFallback(this.#options.home, served); + } + + // The verified manifest, or undefined when there is none to offer. In + // global the release feed comes first, exactly as without npm, and the npm + // packages are asked only when it cannot be reached; in cn the npm + // packages come first, the China mirror's before npm's. Every source's + // manifest must carry the built-in key's signature. + async #manifest( + feed: string, + route: Route, + ): Promise { + const release = new URL(feed); + if (!route.npm) return this.#releaseManifest(feed); + const registries = orderByRegion(route.region, NPM_REGISTRIES); + if (route.region === "global") { + try { + return await this.#releaseManifest(feed); + } catch (error) { + // A manifest the key refuses is no network failure. + if (error instanceof ManifestError) throw error; + this.#failed(route, release, error); + for (const registry of registries) { + try { + const manifest = await this.#npmManifest(registry); + if (!manifest) continue; + await this.#served(route, new URL(registry)); + return manifest; + } catch (npmError) { + this.#failed(route, new URL(registry), npmError); + } + } + throw error; + } + } + // npm can lag the release (a publish that failed or was skipped, a + // stalled mirror sync), so a source with nothing newer than this app + // passes to the next one, ending with the release itself. The first + // source offering a newer version wins. + let first: unknown; + let current: UpdateManifest | undefined; + for (const registry of registries) { + try { + const manifest = await this.#npmManifest(registry); + if (!manifest) continue; + if (!this.#offers(manifest)) { + current ??= manifest; + continue; + } + route.fromNpm = true; + await this.#served(route, new URL(registry)); + return manifest; + } catch (error) { + first ??= error; + this.#failed(route, new URL(registry), error); + } + } + let manifest: UpdateManifest | undefined; + try { + manifest = await this.#releaseManifest(feed); + } catch (error) { + // An npm source that answered says this app is current: the release + // being unreachable from China is no failure then. + if (current) { + this.#failed(route, release, error); + return current; + } + throw first ?? error; + } + if (manifest && this.#offers(manifest)) { + await this.#served(route, release); + return manifest; + } + // Nothing to offer is quiet only when no source failed; a failure is + // retried sooner than a quiet check. + if (!manifest && !current && first !== undefined) throw first; + return manifest ?? current; + } + + // Whether `manifest` names a newer version with a package for this app. + #offers(manifest: UpdateManifest): boolean { + return ( + compareVersions(manifest.version, this.#options.version) > 0 && + packageFor(manifest, this.#options.platform, this.#options.arch) !== + undefined + ); + } + + // manifest.json and its .sig from the latest release of the feed. + async #releaseManifest(feed: string): Promise { + const urls = feedUrls(feed); + const [bytes, signature] = await Promise.all([ + this.#get(urls.manifest), + this.#get(urls.signature), + ]); + // The newest release was published without a manifest: nothing to + // offer until one is. + if (!bytes || !signature) { + this.#ctx!.log.info("update.no_manifest", { feed }); + return undefined; + } + return verifyManifest( + bytes, + signature.toString("utf8"), + this.#options.publicKey ?? UPDATE_PUBLIC_KEY, + ); + } + + // The manifest of the update package's `latest` version on `registry`; + // undefined while the registry has none. + async #npmManifest(registry: string): Promise { + const tags = await this.#get(distTagsUrl(registry, UPDATE_NPM_PACKAGE)); + if (!tags) return undefined; + let latest: unknown; + try { + latest = (JSON.parse(tags.toString("utf8")) as { latest?: unknown }) + .latest; + } catch { + throw new UpdateError("The npm dist-tags are not JSON"); + } + if (!isReleaseVersion(latest)) + throw new UpdateError("The npm dist-tags name no version"); + // A prerelease is never tagged latest on purpose, but npm tags the first + // version of a package latest whatever it is. + if (latest.includes("-")) return undefined; + const url = npmTarballUrl(registry, UPDATE_NPM_PACKAGE, latest); + const response = await (this.#options.fetch ?? fetch)(url); + if (response.status === 404) return undefined; + if (!response.ok || !response.body) + throw new UpdateError(`${url.pathname} returned ${response.status}`); + const files: Record = {}; + await extractNpmTarball( + Readable.fromWeb(response.body as never), + { + [NPM_MANIFEST_MEMBER]: MANIFEST_MAX, + [NPM_SIGNATURE_MEMBER]: SIGNATURE_MAX, + }, + (member, chunk) => { + (files[member] ??= []).push(chunk); + }, + ); + const manifest = verifyManifest( + Buffer.concat(files[NPM_MANIFEST_MEMBER] ?? []), + Buffer.concat(files[NPM_SIGNATURE_MEMBER] ?? []).toString("utf8"), + this.#options.publicKey ?? UPDATE_PUBLIC_KEY, + ); + // An older signed manifest under a newer tag is refused here; one under + // its own older tag is only never newer than this app. + if (manifest.version !== latest) + throw new ManifestError("Update manifest does not match its npm version"); + return manifest; + } + + // The package from the release, or joined from its npm parts. In global + // the release comes first and the parts only when it fails; in cn the + // parts come first, and the release when any part fails on every registry, + // unless the manifest came from the release, whose version npm lacks. + async #downloadPackage( + feed: string, + version: string, + pkg: UpdatePackage, + file: string, + route: Route, + ): Promise { + const url = packageUrl(feed, version, pkg.file); + const release = () => this.#download(url, file, pkg); + if (!route.npm || !pkg.parts) return release(); + if (route.region === "cn" && !route.fromNpm) { + await release(); + return; + } + const parts = () => + this.#downloadParts( + version, + `${this.#options.platform}-${this.#options.arch}`, + pkg, + file, + route, + ); + if (route.region === "global") { + try { + await release(); + } catch (error) { + this.#failed(route, url, error); + try { + await parts(); + } catch { + throw error; + } + } + return; + } + try { + await parts(); + return; + } catch (error) { + try { + await release(); + } catch { + throw error; + } + } + await this.#served(route, url); + } + + // Each part from the first registry that serves it, checked against the + // manifest, then joined into `file` and checked as a whole. + async #downloadParts( + version: string, + target: string, + pkg: UpdatePackage, + file: string, + route: Route, + ): Promise { + const parts = pkg.parts!; + if (parts.reduce((total, part) => total + part.size, 0) !== pkg.size) + throw new UpdateError("Update parts do not add up to the package"); + await mkdir(path.dirname(file), { recursive: true, mode: 0o700 }); + const registries = orderByRegion(route.region, NPM_REGISTRIES); + const pieces: string[] = []; + try { + for (const [index, part] of parts.entries()) { + const piece = `${file}.part${index + 1}`; + pieces.push(piece); + let first: unknown; + let served = false; + for (const registry of registries) { + const url = npmTarballUrl( + registry, + partNpmPackage(target, index), + version, + ); + try { + await this.#downloadPart(url, piece, part); + await this.#served(route, url); + served = true; + break; + } catch (error) { + first ??= error; + this.#failed(route, url, error); + } + } + if (!served) throw first; + } + // Each piece is removed once it is joined, so the disk holds about one + // package's worth at a time. + async function* joined(): AsyncGenerator { + for (const piece of pieces) { + for await (const chunk of createReadStream(piece)) + yield chunk as Buffer; + await rm(piece, { force: true }); + } + } + await this.#verify( + Readable.from(joined()), + pkg, + createWriteStream(file, { mode: 0o600 }), + ); + } catch (error) { + await rm(file, { force: true }); + throw error; + } finally { + for (const piece of pieces) await rm(piece, { force: true }); + } + } + + async #downloadPart( + url: URL, + piece: string, + part: { readonly sha256: string; readonly size: number }, + ): Promise { + const response = await (this.#options.fetch ?? fetch)(url); + if (response.status === 404) { + await response.body?.cancel(); + throw new UpdateNotFound(`${url.pathname} returned 404`); + } + if (!response.ok || !response.body) + throw new UpdateError(`${url.pathname} returned ${response.status}`); + const hash = createHash("sha256"); + let received = 0; + const handle = await open(piece, "w", 0o600); + try { + await extractNpmTarball( + Readable.fromWeb(response.body as never), + { [NPM_PART_MEMBER]: part.size }, + async (_member, chunk) => { + hash.update(chunk); + received += chunk.length; + await handle.write(chunk); + }, + ); + } finally { + await handle.close(); + } + if (received !== part.size || hash.digest("hex") !== part.sha256) + throw new UpdateError("Update part does not match its checksum"); + } + // Follows redirects: GitHub sends release files from another host. The // signature and the checksums, not the host, decide what is trusted. // Undefined when the file does not exist. diff --git a/docs/desktop-release.md b/docs/desktop-release.md index 9ad489837..6a597e123 100644 --- a/docs/desktop-release.md +++ b/docs/desktop-release.md @@ -71,18 +71,19 @@ folder, read only by the signing tools and removed after the build. Set these under Settings → Secrets and variables → Actions of the repository that runs the workflow: -| Name | Kind | Required | Purpose | -| ------------------------------- | -------------------- | ----------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `LINGER_NATIVE_DEPS_URL` | secret (or variable) | no | Folder serving the archives `apps/desktop/distribution/*.json` pins; the `native-deps` prerelease when unset. Each download is checked against its pin. | -| `LINGER_DIAGNOSTICS_UPLOAD_URL` | secret | no | https URL of the hosted diagnostics receiver ([diagnostics-upload.md](diagnostics-upload.md)) that the build carries. A secret, so the address stays masked in logs. Without it Send Diagnostics is hidden unless `config.json` names a receiver. | -| `RELEASE_REPO` | variable | no | `owner/name` that receives the release; `simplexai-labs/linger` when unset. | -| `LINGER_RELEASE_TOKEN` | secret | when `RELEASE_REPO` is another repository | Fine-grained token for `RELEASE_REPO` only, with Contents: read and write. Not needed in simplexai-labs/linger itself. | -| `MACOS_DEVELOPER_ID_P12_BASE64` | secret | no | Base64 of the Developer ID `.p12`. Without it the app is signed ad hoc. | -| `MACOS_DEVELOPER_ID_PASSWORD` | secret | with the `.p12` | Password of that `.p12`. | -| `MACOS_NOTARY_KEY_JSON` | secret | no | App Store Connect API key JSON; needs the Developer ID. | -| `DESKTOP_UPDATE_KEY_PEM` | secret | for auto-update | Ed25519 private key (PEM) that signs `manifest.json`. Without it the release has no manifest and installed apps are not offered it. | -| `WINDOWS_SIGNING_P12_BASE64` | secret | no | Base64 of an Authenticode `.p12`. Without it the setup program is unsigned. | -| `WINDOWS_SIGNING_PASSWORD` | secret | with the `.p12` | Password of that `.p12`. | +| Name | Kind | Required | Purpose | +| ------------------------------- | -------------------- | ----------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `LINGER_NATIVE_DEPS_URL` | secret (or variable) | no | Folder serving the archives `apps/desktop/distribution/*.json` pins; the `native-deps` prerelease when unset. Each download is checked against its pin. | +| `LINGER_DIAGNOSTICS_UPLOAD_URL` | secret | no | https URL of the hosted diagnostics receiver ([diagnostics-upload.md](diagnostics-upload.md)) that the build carries. A secret, so the address stays masked in logs. Without it Send Diagnostics is hidden unless `config.json` names a receiver. | +| `RELEASE_REPO` | variable | no | `owner/name` that receives the release; `simplexai-labs/linger` when unset. | +| `LINGER_RELEASE_TOKEN` | secret | when `RELEASE_REPO` is another repository | Fine-grained token for `RELEASE_REPO` only, with Contents: read and write. Not needed in simplexai-labs/linger itself. | +| `MACOS_DEVELOPER_ID_P12_BASE64` | secret | no | Base64 of the Developer ID `.p12`. Without it the app is signed ad hoc. | +| `MACOS_DEVELOPER_ID_PASSWORD` | secret | with the `.p12` | Password of that `.p12`. | +| `MACOS_NOTARY_KEY_JSON` | secret | no | App Store Connect API key JSON; needs the Developer ID. | +| `DESKTOP_UPDATE_KEY_PEM` | secret | for auto-update | Ed25519 private key (PEM) that signs `manifest.json`. Without it the release has no manifest and installed apps are not offered it. | +| `WINDOWS_SIGNING_P12_BASE64` | secret | no | Base64 of an Authenticode `.p12`. Without it the setup program is unsigned. | +| `WINDOWS_SIGNING_PASSWORD` | secret | with the `.p12` | Password of that `.p12`. | +| `NPM_TOKEN` | secret | for the China update feed | npm automation token that may publish `@simplexai/linger-desktop-*`. Without it the version is not published to npm; installed apps in China then find nothing newer there and take it from the GitHub release ([China update feed](#china-update-feed)). | ### Bundled archives @@ -130,6 +131,9 @@ Each package bundles the source package, PostgreSQL, voice tools, TeamHub and the Native services as described in [Packaging Details](#packaging-details). +With the update key and `NPM_TOKEN`, the same version also goes to npm as the +[China update feed](#china-update-feed). + ## Auto-update Installed apps update from the GitHub Releases themselves @@ -153,7 +157,12 @@ with its size and SHA-256: "darwin-arm64": { "file": "Linger-0.16.36-macos-arm64.app.tar.gz", "sha256": "…", - "size": 123 + "size": 300000123, + "parts": [ + { "sha256": "…", "size": 150000000 }, + { "sha256": "…", "size": 150000000 }, + { "sha256": "…", "size": 123 } + ] }, "linux-x64": { "file": "Linger-0.16.36-linux-x64.tar.gz", @@ -169,8 +178,11 @@ with its size and SHA-256: } ``` -The app downloads the package from `releases/download/desktop-v/` and -checks its size and SHA-256 while downloading. What happens next depends on +`parts` pins the pieces of the package that the +[China update feed](#china-update-feed) publishes; apps from before it ignore +the key. The app downloads the package from +`releases/download/desktop-v/` and checks its size and SHA-256 while +downloading. What happens next depends on how Linger was installed: | Install | Update | @@ -205,7 +217,68 @@ them to its source package (see LocalServer in `updateFeedUrl` in the desktop config (`config.json` in the app's data folder) points the app at another releases URL with the same layout, such as `https://github.com///releases`; it must be https and its -manifest must be signed by the same key. +manifest must be signed by the same key. Such a feed is the only source: the +npm packages below mirror the default feed alone. + +### China update feed + +GitHub is slow or unreachable from mainland China, and the packages are +350–400 MB. So every release with a manifest is also published to npm, which +the npmmirror registry in China mirrors, and apps whose Linger home resolves +to the `cn` download region (`packages/local-installation/src/region.ts`, +`SIMPLEX_REGION` or the registry probe, recorded in `runtime/region.json`) +take their updates from there: + +| npm package | Holds | +| ------------------------------------------------------ | -------------------------------------------------------------------------- | +| `@simplexai/linger-desktop-update@` | `manifest.json` and `manifest.json.sig`, byte for byte as in the release | +| `@simplexai/linger-desktop--part@` | `part`: piece N (from 1) of the package of ``, e.g. `darwin-arm64` | + +`desktop-release.sh manifest` cuts each package into pieces of at most +150,000,000 bytes (`DESKTOP_NPM_PART_BYTES`; npmmirror has served tarballs up +to 172 MB and documents no limit) and pins each piece's size and SHA-256 in +`parts`, so the signature covers them. `npm-pack` then cuts the pieces at those +sizes, checks each against the manifest and packs the packages with `npm pack`; +`npm-publish --publish` publishes them with `NPM_TOKEN`, the parts before the +manifest, under the dist-tag `latest` for a stable version and `next` for a +prerelease (matching the GitHub latest release), skips versions npm already +has, and asks npmmirror to sync each package +(`PUT https://registry-direct.npmmirror.com/-/package//syncs`). Without +`--publish` it only lists what it would publish. The `release` job runs both +after the GitHub Release when `NPM_TOKEN` is set. + +In `cn` an app checks: + +1. the `latest` dist-tag of `@simplexai/linger-desktop-update` on + `registry.npmmirror.com`, then that version's tarball from + `cdn.npmmirror.com`; then the same on `registry.npmjs.org`; then the GitHub + release. The first source whose manifest names a version newer than the app + (with a package for it) wins; a source that is behind, because npm was not + published, the mirror has not synced or a source failed, passes to the next, + so a release npm never got still reaches China from GitHub. A prerelease + under `latest` (npm tags a package's first version `latest` whatever it is) + offers nothing. Whatever the source, the manifest must carry the signature + of `UPDATE_PUBLIC_KEY` and name the tagged version. When npm answered that + the app is current, an unreachable GitHub is no failure. +2. for a manifest from npm, each part's tarball, from `cdn.npmmirror.com` and + then `registry.npmjs.org`, checking the part's size and SHA-256; the parts + are joined and checked as the whole package. When a part fails on both, it + downloads the whole package from the GitHub release instead. For a + manifest only the GitHub release had, the package comes from there. + +The app unpacks only the expected files from a tarball (`update-npm.ts`: a +regular `package/part`, or `package/manifest.json` and its `.sig`, plus npm's +`package/package.json`, each within its size) and rejects anything else. +npmmirror's per-file endpoint needs a whitelist our packages are not on, so +the app reads the tarball itself. In `global` the app checks GitHub first, +exactly as described above, and asks npm (`registry.npmjs.org`, then +npmmirror) only when GitHub cannot be reached. When the other region serves +what the home's region could not, other than a file the mirror has not synced +yet, the home's region becomes that one (`recordRegionFallback`), for the +installer's downloads too. + +npmmirror syncs a new version minutes after it is published; until then apps +in China find the previous version, or take a missing part from npm. ### Signing key @@ -234,8 +307,13 @@ DESKTOP_VERSION=0.16.36-rc.1 RCODESIGN=/path/to/rcodesign \ DESKTOP_UPDATE_KEY=/path/to/ed25519.pem \ scripts/ci/desktop-release.sh manifest .linger-runtime/desktop-release/assets 0.16.36-rc.1 scripts/ci/desktop-release.sh checksums .linger-runtime/desktop-release/assets 0.16.36-rc.1 +scripts/ci/desktop-release.sh npm-pack .linger-runtime/desktop-release/assets 0.16.36-rc.1 \ + .linger-runtime/desktop-release/npm +scripts/ci/desktop-release.sh npm-publish .linger-runtime/desktop-release/npm 0.16.36-rc.1 ``` +`npm-publish` publishes only with `--publish` and `NPM_TOKEN` set. + With the archives already in `LINGER_NATIVE_POSTGRES_DIR` (default `~/.cache/linger-desktop`), `LINGER_NATIVE_DEPS_URL` is not needed. `manifest` needs the macOS app archive, the Linux archive and the setup program diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 418d67624..159e3eb65 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -144,6 +144,9 @@ importers: '@linger/local-installation': specifier: workspace:* version: link:../../packages/local-installation + '@linger/service-kit': + specifier: workspace:* + version: link:../../packages/service-kit '@linger/wire-contracts': specifier: workspace:* version: link:../../contracts/generated/typescript diff --git a/scripts/ci/desktop-npm-pack.mjs b/scripts/ci/desktop-npm-pack.mjs new file mode 100644 index 000000000..c39122e5a --- /dev/null +++ b/scripts/ci/desktop-npm-pack.mjs @@ -0,0 +1,141 @@ +#!/usr/bin/env node +// Packs a release's China update feed as npm packages (desktop-release.sh +// npm-pack): @simplexai/linger-desktop-update with the signed manifest.json +// and manifest.json.sig, and each package the manifest splits into `parts` as +// @simplexai/linger-desktop--part, one file `part` each. The +// parts are cut at the sizes the signed manifest pins, and each must match +// its sha256 there, so the packages hold exactly what installed apps accept +// (apps/desktop/src/main/update-npm.ts reads them). +// +// node desktop-npm-pack.mjs +// +// Writes the .tgz files into and packages.txt, one +// " " line per package in publishing order: the parts first, +// so the manifest naming them is published last. + +import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { + closeSync, + existsSync, + mkdirSync, + openSync, + readFileSync, + readSync, + readdirSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; +import path from "node:path"; +import process from "node:process"; + +const UPDATE_PACKAGE = "@simplexai/linger-desktop-update"; +const REPOSITORY = "github:simplexai-labs/linger"; +const LICENSE = + "SEE LICENSE IN https://github.com/simplexai-labs/linger/blob/main/LICENSE"; + +const fail = (message) => { + console.error(`desktop-npm-pack: ${message}`); + process.exit(1); +}; + +const sha256 = (bytes) => createHash("sha256").update(bytes).digest("hex"); + +const [dir, version, out] = process.argv.slice(2); +if (!dir || !version || !out) + fail("usage: desktop-npm-pack.mjs "); +if (path.resolve(out) === path.resolve(dir)) + fail("the out folder must not be the release folder"); +if (existsSync(out) && readdirSync(out).length > 0) fail(`${out} is not empty`); + +let manifest; +try { + manifest = JSON.parse(readFileSync(path.join(dir, "manifest.json"), "utf8")); +} catch { + fail(`${dir} has no readable manifest.json`); +} +if (!existsSync(path.join(dir, "manifest.json.sig"))) + fail(`${dir} has no manifest.json.sig`); +if (manifest.schema !== 2 || manifest.version !== version) + fail(`manifest.json is not the schema 2 manifest of ${version}`); + +mkdirSync(out, { recursive: true }); +const staging = path.join(out, ".staging"); +const published = []; + +// One package from `files` (name -> bytes), packed with npm into `out`. +const pack = (name, description, files) => { + rmSync(staging, { recursive: true, force: true }); + mkdirSync(staging, { recursive: true }); + const packageJson = { + name, + version, + description, + license: LICENSE, + repository: REPOSITORY, + files: Object.keys(files), + }; + writeFileSync( + path.join(staging, "package.json"), + `${JSON.stringify(packageJson, null, 2)}\n`, + ); + for (const [file, bytes] of Object.entries(files)) + writeFileSync(path.join(staging, file), bytes); + const [result] = JSON.parse( + execFileSync( + "npm", + [ + "pack", + "--json", + "--ignore-scripts", + "--pack-destination", + path.resolve(out), + ], + { cwd: staging, encoding: "utf8", stdio: ["ignore", "pipe", "inherit"] }, + ), + ); + rmSync(staging, { recursive: true, force: true }); + published.push(`${name} ${result.filename}`); + console.log(`desktop-npm-pack: packed ${name}@${version}`); +}; + +for (const [target, pkg] of Object.entries(manifest.packages).sort()) { + if (!/^[a-z0-9]+-[a-z0-9]+$/.test(target)) + fail(`${target} is not a platform-arch target`); + if (!Array.isArray(pkg.parts) || pkg.parts.length === 0) + fail(`manifest.json lists no parts for ${target}`); + const file = path.join(dir, pkg.file); + if (!existsSync(file)) fail(`${dir} has no ${pkg.file}`); + if (statSync(file).size !== pkg.size) + fail(`${pkg.file} does not match manifest.json`); + const whole = createHash("sha256"); + const fd = openSync(file, "r"); + let offset = 0; + try { + for (const [index, part] of pkg.parts.entries()) { + const bytes = Buffer.alloc(part.size); + if (readSync(fd, bytes, 0, part.size, offset) !== part.size) + fail(`${pkg.file} is shorter than its parts`); + offset += part.size; + if (sha256(bytes) !== part.sha256) + fail(`part ${index + 1} of ${pkg.file} does not match manifest.json`); + whole.update(bytes); + pack( + `@simplexai/linger-desktop-${target}-part${index + 1}`, + `Part ${index + 1} of ${pkg.parts.length} of ${pkg.file}, checked by the signed manifest in ${UPDATE_PACKAGE}@${version}.`, + { part: bytes }, + ); + } + } finally { + closeSync(fd); + } + if (offset !== pkg.size || whole.digest("hex") !== pkg.sha256) + fail(`${pkg.file} does not match manifest.json`); +} + +pack(UPDATE_PACKAGE, `Signed update manifest of Linger desktop ${version}.`, { + "manifest.json": readFileSync(path.join(dir, "manifest.json")), + "manifest.json.sig": readFileSync(path.join(dir, "manifest.json.sig")), +}); +writeFileSync(path.join(out, "packages.txt"), `${published.join("\n")}\n`); diff --git a/scripts/ci/desktop-release.sh b/scripts/ci/desktop-release.sh index 385a4a49f..afb45f3c0 100755 --- a/scripts/ci/desktop-release.sh +++ b/scripts/ci/desktop-release.sh @@ -20,6 +20,13 @@ set -euo pipefail # verify stack pins and copy install.sh # and install.ps1 into # desktop-release.sh checksums write /SHA256SUMS.txt +# desktop-release.sh npm-pack +# pack the China update feed from 's +# signed manifest: npm tarballs into +# desktop-release.sh npm-publish [--publish] +# list what npm-pack made; with --publish +# (and NPM_TOKEN) publish it to npm and ask +# npmmirror to sync it # desktop-release.sh gitlab-upload # upload 's files to a GitLab generic # package from a GitLab CI job (test builds only) @@ -45,6 +52,8 @@ set -euo pipefail # MACOS_NOTARY_KEY App Store Connect API key JSON; notarizes and staples # DESKTOP_UPDATE_KEY Ed25519 PEM file that signs the update manifest; without # it the release has no manifest and offers no update +# DESKTOP_NPM_PART_BYTES largest npm part of a package (150000000) +# NPM_TOKEN npm automation token with publish rights to @simplexai # WINDOWS_SIGNING_P12, WINDOWS_SIGNING_PASSWORD_FILE # Authenticode .p12 and a file with its password for # the setup program (osslsigncode); unsigned without @@ -71,7 +80,8 @@ die() { usage() { echo "Usage: $0 version | fetch-native|compile|package|build " \ "| manifest|install-scripts|checksums | github-release " \ - "| gitlab-upload " >&2 + "| gitlab-upload | npm-pack " \ + "| npm-publish [--publish]" >&2 exit 2 } @@ -279,13 +289,23 @@ cmd_build() { # portable Windows folder only link to the release. update_packages='darwin-arm64:macos-arm64.app.tar.gz linux-x64:linux-x64.tar.gz win32-x64:windows-x64-setup.exe' +# The largest part of a package published to npm. npmmirror has served +# tarballs of 172 MB; it documents no limit, so parts stay below that. +part_bytes() { + local bytes="${DESKTOP_NPM_PART_BYTES:-150000000}" + [[ "$bytes" =~ ^[1-9][0-9]*$ ]] || die "DESKTOP_NPM_PART_BYTES must be a byte count" + echo "$bytes" +} + # manifest.json lists every platform's package with its size and sha256, and -# manifest.json.sig is its Ed25519 signature, made with DESKTOP_UPDATE_KEY. +# the size and sha256 of each part of it that npm-pack publishes (at most +# DESKTOP_NPM_PART_BYTES each). manifest.json.sig is its Ed25519 signature, +# made with DESKTOP_UPDATE_KEY. # Installed apps read both from the latest release and download the package # from the release of the version it names. Without the key the release goes # out without them, and no installed app is offered it. cmd_manifest() { - local dir="${1:-}" version="${2:-}" entry target suffix packages=() + local dir="${1:-}" version="${2:-}" entry target suffix part_size packages=() [[ -d "$dir" ]] || die "manifest needs the folder holding the release files" [[ "$version" =~ $version_pattern ]] || die "manifest needs the release version" rm -f "$dir/manifest.json" "$dir/manifest.json.sig" @@ -294,6 +314,7 @@ cmd_manifest() { return 0 fi [[ -f "$DESKTOP_UPDATE_KEY" ]] || die "DESKTOP_UPDATE_KEY names no file" + part_size="$(part_bytes)" for entry in $update_packages; do target="${entry%%:*}" suffix="${entry#*:}" @@ -302,18 +323,25 @@ cmd_manifest() { done # The key file is read only here; the built-in public key decides whether # installed apps would accept the signature. - node - "$dir" "$version" "$DESKTOP_UPDATE_KEY" "$desktop/src/main/update-manifest.ts" "${packages[@]}" <<'NODE' || + node - "$dir" "$version" "$DESKTOP_UPDATE_KEY" "$desktop/src/main/update-manifest.ts" "$part_size" "${packages[@]}" <<'NODE' || const crypto = require("crypto"); const fs = require("fs"); const path = require("path"); -const [dir, version, keyFile, source, ...pairs] = process.argv.slice(2); +const [dir, version, keyFile, source, partBytes, ...pairs] = process.argv.slice(2); +const sha256 = (bytes) => crypto.createHash("sha256").update(bytes).digest("hex"); const packages = {}; for (let i = 0; i < pairs.length; i += 2) { const bytes = fs.readFileSync(path.join(dir, pairs[i + 1])); + const parts = []; + for (let start = 0; start < bytes.length; start += Number(partBytes)) { + const part = bytes.subarray(start, start + Number(partBytes)); + parts.push({ sha256: sha256(part), size: part.length }); + } packages[pairs[i]] = { file: pairs[i + 1], - sha256: crypto.createHash("sha256").update(bytes).digest("hex"), + sha256: sha256(bytes), size: bytes.length, + parts, }; } const manifest = Buffer.from( @@ -369,6 +397,68 @@ cmd_checksums() { cat "$dir/SHA256SUMS.txt" } +# The China update feed (docs/desktop-release.md#china-update-feed): the +# signed manifest and every package's parts as npm packages, which npmmirror +# mirrors. must be outside , whose files all go to the release. +cmd_npm_pack() { + local dir="${1:-}" version="${2:-}" out="${3:-}" + [[ -d "$dir" && -n "$out" && "$version" =~ $version_pattern ]] || + die "npm-pack needs the folder of release files, the release version and an out folder" + [[ -f "$dir/manifest.json" && -f "$dir/manifest.json.sig" ]] || + die "npm-pack needs the signed manifest; run '$0 manifest $dir $version' with DESKTOP_UPDATE_KEY" + command -v npm >/dev/null || die "npm-pack needs npm" + node "$repo_root/scripts/ci/desktop-npm-pack.mjs" "$dir" "$version" "$out" +} + +# Publishes what npm-pack made only with --publish; otherwise it lists it. +# npm's dist-tag follows the GitHub release: latest for a stable version, +# next for a prerelease. A version npm already has is skipped, so a retried +# job goes on where it stopped. npmmirror then syncs each package; until it +# has, installed apps in China take them from npm or GitHub. +cmd_npm_publish() { + local out="${1:-}" version="${2:-}" mode="${3:-}" tag=latest name tarball names=() + [[ -f "$out/packages.txt" && "$version" =~ $version_pattern ]] || + die "npm-publish needs the out folder of '$0 npm-pack' and the release version" + [[ -z "$mode" || "$mode" == --publish ]] || die "npm-publish takes only --publish" + [[ "$version" == *-* ]] && tag=next + while read -r name tarball <&3; do + [[ -n "$name" ]] || continue + [[ -f "$out/$tarball" ]] || die "$out has no $tarball" + names+=("$name") + [[ "$mode" == --publish ]] || echo "desktop-release: would publish $name@$version ($tag) from $tarball" + done 3<"$out/packages.txt" + if [[ "$mode" != --publish ]]; then + echo "desktop-release: nothing published; pass --publish with NPM_TOKEN set to publish" + return 0 + fi + [[ -n "${NPM_TOKEN:-}" ]] || die "npm-publish --publish needs NPM_TOKEN" + command -v npm >/dev/null || die "npm-publish needs npm" + # npm reads the token from the environment; it is never written down. + # Global, since the trap runs after this function returns. + npmrc="$(mktemp)" + trap 'rm -f "$npmrc"' EXIT + # shellcheck disable=SC2016 + printf '%s\n' '//registry.npmjs.org/:_authToken=${NPM_TOKEN}' >"$npmrc" + while read -r name tarball <&3; do + [[ -n "$name" ]] || continue + if [[ -n "$(npm view "$name@$version" version --userconfig "$npmrc" --registry https://registry.npmjs.org/ 2>/dev/null)" ]]; then + echo "desktop-release: $name@$version is already on npm" + continue + fi + npm publish "$out/$tarball" --access public --tag "$tag" \ + --userconfig "$npmrc" --registry https://registry.npmjs.org/ /dev/null; then + echo "desktop-release: asked npmmirror to sync $name" + else + echo "::warning::npmmirror did not take the sync of $name; it also syncs new versions on its own, later." + fi + done +} + gitlab_env() { local name for name in CI_API_V4_URL CI_PROJECT_ID CI_JOB_TOKEN; do @@ -448,6 +538,8 @@ case "$command" in manifest) cmd_manifest "$@" ;; install-scripts) cmd_install_scripts "$@" ;; checksums) cmd_checksums "$@" ;; + npm-pack) cmd_npm_pack "$@" ;; + npm-publish) cmd_npm_publish "$@" ;; gitlab-upload) cmd_gitlab_upload "$@" ;; github-release) cmd_github_release "$@" ;; *) usage ;; diff --git a/scripts/ci/desktop-release.test.ts b/scripts/ci/desktop-release.test.ts index 09ee0a67a..8dd894ff3 100644 --- a/scripts/ci/desktop-release.test.ts +++ b/scripts/ci/desktop-release.test.ts @@ -16,6 +16,12 @@ import { join } from "node:path"; import { afterEach, describe, expect, it } from "vitest"; +import { + UPDATE_NPM_PACKAGE, + extractNpmTarball, + partNpmPackage, +} from "../../apps/desktop/src/main/update-npm.ts"; + const script = join(process.cwd(), "scripts/ci/desktop-release.sh"); const temporary: string[] = []; @@ -191,6 +197,12 @@ describe("desktop-release.sh manifest", () => { file, sha256: createHash("sha256").update(file).digest("hex"), size: file.length, + parts: [ + { + sha256: createHash("sha256").update(file).digest("hex"), + size: file.length, + }, + ], }, ]), ), @@ -202,6 +214,33 @@ describe("desktop-release.sh manifest", () => { expect(sums).toContain(" manifest.json.sig\n"); }); + it("pins the parts of each package npm-pack publishes", () => { + const dir = release(); + const repo = checkout(); + + const result = run( + ["manifest", dir, version], + { DESKTOP_UPDATE_KEY: repo.key, DESKTOP_NPM_PART_BYTES: "16" }, + repo.script, + ); + + expect(result.status).toBe(0); + const manifest = JSON.parse( + readFileSync(join(dir, "manifest.json"), "utf8"), + ) as { packages: Record }; + // Linger-0.16.33-windows-x64-setup.exe is 36 bytes. + expect( + manifest.packages["win32-x64"]!.parts.map((part) => part.size), + ).toEqual([16, 16, 4]); + expect( + run( + ["manifest", dir, version], + { DESKTOP_UPDATE_KEY: repo.key, DESKTOP_NPM_PART_BYTES: "0" }, + repo.script, + ).stderr, + ).toContain("DESKTOP_NPM_PART_BYTES"); + }); + it("warns and writes nothing without the key", () => { const dir = release(); @@ -454,3 +493,202 @@ describe("desktop-release.sh install-scripts", () => { expect(existsSync(join(dir, "install.sh"))).toBe(false); }); }); + +describe("desktop-release.sh npm-pack and npm-publish", () => { + const version = "0.16.33"; + const packages = { + "darwin-arm64": `Linger-${version}-macos-arm64.app.tar.gz`, + "linux-x64": `Linger-${version}-linux-x64.tar.gz`, + "win32-x64": `Linger-${version}-windows-x64-setup.exe`, + }; + // A release of the three packages, 40 bytes each, signed with parts of + // at most 16 bytes by a throwaway key. + const signedRelease = () => { + const { privateKey, publicKey } = generateKeyPairSync("ed25519"); + const root = folder(); + mkdirSync(join(root, "scripts/ci"), { recursive: true }); + mkdirSync(join(root, "apps/desktop/src/main"), { recursive: true }); + copyFileSync(script, join(root, "scripts/ci/desktop-release.sh")); + writeFileSync( + join(root, "apps/desktop/src/main/update-manifest.ts"), + `export const UPDATE_PUBLIC_KEY = \`${publicKey.export({ type: "spki", format: "pem" }).toString().trim()}\`;\n`, + ); + const key = join(folder(), "update.pem"); + writeFileSync( + key, + privateKey.export({ type: "pkcs8", format: "pem" }).toString(), + ); + const dir = folder(); + for (const [target, name] of Object.entries(packages)) + writeFileSync(join(dir, name), `${target}:`.padEnd(40, target[0])); + const signed = run( + ["manifest", dir, version], + { DESKTOP_UPDATE_KEY: key, DESKTOP_NPM_PART_BYTES: "16" }, + join(root, "scripts/ci/desktop-release.sh"), + ); + expect(signed.status).toBe(0); + return dir; + }; + + // The members of a packed tarball, read as installed apps read them. + const extract = async (tarball: string, ...members: string[]) => { + const chunks: Record = {}; + await extractNpmTarball( + [readFileSync(tarball)], + Object.fromEntries(members.map((member) => [member, 1024 * 1024])), + (member, chunk) => { + (chunks[member] ??= []).push(Buffer.from(chunk)); + }, + ); + return Object.fromEntries( + Object.entries(chunks).map(([member, parts]) => [ + member, + Buffer.concat(parts), + ]), + ); + }; + + it("packs the manifest and every part the way installed apps read them", async () => { + const dir = signedRelease(); + const out = join(folder(), "npm"); + + const result = run(["npm-pack", dir, version, out]); + + expect(result.status, result.stderr).toBe(0); + const lines = readFileSync(join(out, "packages.txt"), "utf8") + .trim() + .split("\n") + .map((line) => line.split(" ") as [string, string]); + const targets = Object.keys(packages).sort(); + expect(lines.map(([name]) => name)).toEqual([ + ...targets.flatMap((target) => + [0, 1, 2].map((index) => partNpmPackage(target, index)), + ), + // Last, so no manifest names parts npm does not have yet. + UPDATE_NPM_PACKAGE, + ]); + const tarballs = Object.fromEntries(lines); + for (const [target, file] of Object.entries(packages)) { + const joined = Buffer.concat( + await Promise.all( + [0, 1, 2].map((index) => + extract( + join(out, tarballs[partNpmPackage(target, index)]!), + "package/part", + ).then((members) => members["package/part"]!), + ), + ), + ); + expect(joined).toEqual(readFileSync(join(dir, file))); + } + expect( + await extract( + join(out, tarballs[UPDATE_NPM_PACKAGE]!), + "package/manifest.json", + "package/manifest.json.sig", + ), + ).toEqual({ + "package/manifest.json": readFileSync(join(dir, "manifest.json")), + "package/manifest.json.sig": readFileSync(join(dir, "manifest.json.sig")), + }); + expect(existsSync(join(out, ".staging"))).toBe(false); + }); + + it("refuses a package that changed after the manifest was signed", () => { + const dir = signedRelease(); + writeFileSync(join(dir, packages["linux-x64"]), "x".repeat(40)); + + const result = run(["npm-pack", dir, version, join(folder(), "npm")]); + + expect(result.status).toBe(1); + expect(result.stderr).toContain(packages["linux-x64"]); + }); + + it("needs the signed manifest", () => { + const dir = folder(); + const result = run(["npm-pack", dir, version, join(folder(), "npm")]); + + expect(result.status).toBe(1); + expect(result.stderr).toContain("signed manifest"); + }); + + // An npm and a curl that record their arguments, the npm config they + // were given, and an npm that has no version yet. + const publish = (args: string[], env: Record = {}) => { + const out = folder(); + const bin = folder(); + const log = join(out, "calls.log"); + writeFileSync(join(out, "a.tgz"), ""); + writeFileSync(join(out, "b.tgz"), ""); + writeFileSync( + join(out, "packages.txt"), + "@simplexai/linger-desktop-linux-x64-part1 a.tgz\n@simplexai/linger-desktop-update b.tgz\n", + ); + writeFileSync( + join(bin, "npm"), + `#!/bin/sh\necho "npm $*" >>"${log}"\nprev=\nfor a; do [ "$prev" = --userconfig ] && cat "$a" >>"${log}"; prev=$a; done\nexit 0\n`, + ); + writeFileSync(join(bin, "curl"), `#!/bin/sh\necho "curl $*" >>"${log}"\n`); + chmodSync(join(bin, "npm"), 0o755); + chmodSync(join(bin, "curl"), 0o755); + const result = run(["npm-publish", out, ...args], { + PATH: `${bin}:${process.env.PATH ?? ""}`, + ...env, + }); + let calls = ""; + try { + calls = readFileSync(log, "utf8"); + } catch { + // nothing ran + } + return { ...result, calls }; + }; + + it("only lists the packages without --publish", () => { + const result = publish([version], { NPM_TOKEN: "token" }); + + expect(result.status).toBe(0); + expect(result.stdout).toContain( + `would publish @simplexai/linger-desktop-update@${version} (latest)`, + ); + expect(result.calls).toBe(""); + }); + + it("needs NPM_TOKEN to publish", () => { + const result = publish([version, "--publish"]); + + expect(result.status).toBe(1); + expect(result.stderr).toContain("NPM_TOKEN"); + expect(result.calls).toBe(""); + }); + + it("publishes the parts, then the manifest, and asks npmmirror to sync", () => { + const result = publish([version, "--publish"], { + NPM_TOKEN: "secret-token", + }); + + expect(result.status, result.stderr).toBe(0); + const publishes = result.calls + .split("\n") + .filter((line) => line.startsWith("npm publish")); + expect(publishes).toHaveLength(2); + expect(publishes[0]).toContain("/a.tgz --access public --tag latest"); + expect(publishes[1]).toContain("/b.tgz --access public --tag latest"); + expect(result.calls).toContain( + "curl -fsS --retry 3 -X PUT https://registry-direct.npmmirror.com/-/package/@simplexai/linger-desktop-update/syncs", + ); + // npm expands the token itself; it is never written to a file. + expect(result.calls).toContain( + "//registry.npmjs.org/:_authToken=${NPM_TOKEN}", + ); + expect(result.calls + result.stdout).not.toContain("secret-token"); + }); + + it("tags a prerelease next", () => { + const result = publish(["1.0.0-rc.1", "--publish"], { NPM_TOKEN: "t" }); + + expect(result.status).toBe(0); + expect(result.calls).toContain("--tag next"); + expect(result.calls).not.toContain("--tag latest"); + }); +});