From 5bddf1d19a175a069ecc4e37fa564b72c6e37770 Mon Sep 17 00:00:00 2001 From: Adam Daley Date: Sat, 7 Nov 2026 14:52:45 +0000 Subject: [PATCH 01/12] Prepare and publish extension packages through a queue --- .gitignore | 1 + package-lock.json | 30 + package.json | 9 +- scripts/package-demo.ts | 283 +++ src/app/index.ts | 6 + src/lib/adapters/cloudflare/index.ts | 8 + src/lib/adapters/cloudflare/queue.ts | 9 + src/lib/context.ts | 8 +- src/lib/interfaces.ts | 5 + src/lib/releases.ts | 30 +- src/services/extensions/v1/database.ts | 6 +- src/services/extensions/v1/interfaces.ts | 1 + src/services/extensions/v2/PACKAGE-RUNTIME.md | 38 + src/services/extensions/v2/PACKAGING.md | 260 +++ src/services/extensions/v2/README.md | 14 +- src/services/extensions/v2/db/badges.ts | 4 +- src/services/extensions/v2/db/extensions.ts | 25 +- .../db/migrations/0030_directory_packages.sql | 82 + .../v2/db/migrations/0031_package_jobs.sql | 28 + .../v2/db/migrations/meta/0030_snapshot.json | 1708 +++++++++++++++ .../v2/db/migrations/meta/0031_snapshot.json | 1896 +++++++++++++++++ .../v2/db/migrations/meta/_journal.json | 14 + src/services/extensions/v2/db/package-jobs.ts | 169 ++ src/services/extensions/v2/db/packages.ts | 163 ++ src/services/extensions/v2/db/revisions.ts | 41 +- src/services/extensions/v2/db/schema.ts | 130 ++ src/services/extensions/v2/index.ts | 2 + src/services/extensions/v2/package-builder.ts | 206 ++ .../extensions/v2/packages/b2-store.ts | 130 ++ src/services/extensions/v2/packages/build.ts | 474 +++++ .../v2/packages/cloudflare-queue.ts | 108 + src/services/extensions/v2/packages/errors.ts | 9 + src/services/extensions/v2/packages/jobs.ts | 81 + .../extensions/v2/packages/public-assets.ts | 201 ++ src/services/extensions/v2/packages/store.ts | 162 ++ .../extensions/v2/routes/moderation.ts | 3 + src/services/extensions/v2/routes/packages.ts | 525 +++++ src/services/extensions/v2/schemas/badges.ts | 1 + .../extensions/v2/schemas/extensions.ts | 14 +- .../extensions/v2/schemas/manifest.ts | 8 +- .../extensions/v2/schemas/packages.ts | 52 + test/lib/auth/bearer-assertion.test.ts | 3 + test/lib/releases.test.ts | 15 + test/services/extensions/v2/b2-store.test.ts | 158 ++ test/services/extensions/v2/badges.test.ts | 39 + test/services/extensions/v2/db-fixtures.ts | 7 + test/services/extensions/v2/manifest.test.ts | 1 - .../extensions/v2/package-builder.test.ts | 323 +++ .../extensions/v2/package-jobs.node.test.ts | 108 + test/services/extensions/v2/packages.test.ts | 1467 +++++++++++++ .../extensions/v2/resource-bounds.test.ts | 1 - vitest.config.ts | 5 +- vitest.node.config.ts | 3 +- worker-configuration.d.ts | 163 +- wrangler.jsonc | 119 +- 55 files changed, 9282 insertions(+), 74 deletions(-) create mode 100644 scripts/package-demo.ts create mode 100644 src/lib/adapters/cloudflare/queue.ts create mode 100644 src/services/extensions/v2/PACKAGE-RUNTIME.md create mode 100644 src/services/extensions/v2/PACKAGING.md create mode 100644 src/services/extensions/v2/db/migrations/0030_directory_packages.sql create mode 100644 src/services/extensions/v2/db/migrations/0031_package_jobs.sql create mode 100644 src/services/extensions/v2/db/migrations/meta/0030_snapshot.json create mode 100644 src/services/extensions/v2/db/migrations/meta/0031_snapshot.json create mode 100644 src/services/extensions/v2/db/package-jobs.ts create mode 100644 src/services/extensions/v2/db/packages.ts create mode 100644 src/services/extensions/v2/package-builder.ts create mode 100644 src/services/extensions/v2/packages/b2-store.ts create mode 100644 src/services/extensions/v2/packages/build.ts create mode 100644 src/services/extensions/v2/packages/cloudflare-queue.ts create mode 100644 src/services/extensions/v2/packages/errors.ts create mode 100644 src/services/extensions/v2/packages/jobs.ts create mode 100644 src/services/extensions/v2/packages/public-assets.ts create mode 100644 src/services/extensions/v2/packages/store.ts create mode 100644 src/services/extensions/v2/routes/packages.ts create mode 100644 src/services/extensions/v2/schemas/packages.ts create mode 100644 test/services/extensions/v2/b2-store.test.ts create mode 100644 test/services/extensions/v2/package-builder.test.ts create mode 100644 test/services/extensions/v2/package-jobs.node.test.ts create mode 100644 test/services/extensions/v2/packages.test.ts diff --git a/.gitignore b/.gitignore index 1b94e00d..481f31b5 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ node_modules/ .env .env.production .dev.vars +.dev.vars.* # logs logs/ diff --git a/package-lock.json b/package-lock.json index 4f652633..49f36e78 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,6 +11,9 @@ "@octokit/graphql": "9.0.5", "@octokit/request": "10.0.16", "@scalar/hono-api-reference": "0.12.17", + "@zeropress/svg-hush": "0.1.1", + "@zip.js/zip.js": "2.24.0", + "aws4fetch": "1.0.20", "badge-maker": "6.0.0", "better-auth": "^1.7.14", "drizzle-orm": "0.45.4", @@ -31,6 +34,7 @@ "eslint-config-prettier": "10.1.8", "jiti": "2.7.0", "prettier": "3.9.10", + "tsx": "4.23.15", "typescript": "6.0.3", "typescript-eslint": "8.72.1", "vitest": "5.0.5", @@ -3196,6 +3200,26 @@ "url": "https://opencollective.com/vitest" } }, + "node_modules/@zeropress/svg-hush": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/@zeropress/svg-hush/-/svg-hush-0.1.1.tgz", + "integrity": "sha512-RC19T+8sJVNOJW7iMDqg7SwZfnx9WQ54xRPmzNsj38iq8ZNhibOdA8JAUqZ4FizqTvv1OWvaUmrkNTe1LFxz9Q==", + "license": "MIT", + "engines": { + "node": ">=22.22.0" + } + }, + "node_modules/@zip.js/zip.js": { + "version": "2.24.0", + "resolved": "https://registry.npmjs.org/@zip.js/zip.js/-/zip.js-2.24.0.tgz", + "integrity": "sha512-WlaVKzdqBK+RF3bdiXZWbdCnvGB6PdO4IHC1mn/2Yfl3bASgqUNegnU3+PNg61iecwoPfUi5BgLUPqBukXpyWw==", + "license": "BSD-3-Clause", + "engines": { + "bun": ">=0.7.0", + "deno": ">=1.0.0", + "node": ">=18.0.0" + } + }, "node_modules/acorn": { "version": "8.19.0", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.19.0.tgz", @@ -3255,6 +3279,12 @@ "node": ">=12" } }, + "node_modules/aws4fetch": { + "version": "1.0.20", + "resolved": "https://registry.npmjs.org/aws4fetch/-/aws4fetch-1.0.20.tgz", + "integrity": "sha512-/djoAN709iY65ETD6LKCtyyEI04XIBP5xVvfmNxsEP0uJB5tyaGBztSryRr4HqMStr9R06PisQE7m9zDTXKu6g==", + "license": "MIT" + }, "node_modules/badge-maker": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/badge-maker/-/badge-maker-6.0.0.tgz", diff --git a/package.json b/package.json index 01cde373..798034ff 100644 --- a/package.json +++ b/package.json @@ -22,13 +22,19 @@ "test:node": "vitest run --config vitest.node.config.ts", "test:all": "npm run test && npm run test:node", "test:coverage": "vitest run --coverage", - "typecheck": "tsc --noEmit" + "typecheck": "tsc --noEmit", + "packages:setup": "tsx scripts/package-demo.ts setup", + "packages:dev": "wrangler dev --env packages-local --local --ip 127.0.0.1 --port 8789 --persist-to .wrangler/package-demo", + "packages:demo": "tsx scripts/package-demo.ts exercise" }, "dependencies": { "@hono/zod-openapi": "1.6.3", "@octokit/graphql": "9.0.5", "@octokit/request": "10.0.16", "@scalar/hono-api-reference": "0.12.17", + "@zeropress/svg-hush": "0.1.1", + "@zip.js/zip.js": "2.24.0", + "aws4fetch": "1.0.20", "badge-maker": "6.0.0", "better-auth": "^1.7.14", "drizzle-orm": "0.45.4", @@ -49,6 +55,7 @@ "eslint-config-prettier": "10.1.8", "jiti": "2.7.0", "prettier": "3.9.10", + "tsx": "4.23.15", "typescript": "6.0.3", "typescript-eslint": "8.72.1", "vitest": "5.0.5", diff --git a/scripts/package-demo.ts b/scripts/package-demo.ts new file mode 100644 index 00000000..e89921c5 --- /dev/null +++ b/scripts/package-demo.ts @@ -0,0 +1,283 @@ +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { readFile, writeFile } from "node:fs/promises"; +import { sign } from "hono/jwt"; +import { TextReader, Uint8ArrayWriter, ZipWriter } from "@zip.js/zip.js"; +import { PackageArtifactSchema } from "../src/services/extensions/v2/schemas/packages"; +import { sha256 } from "../src/services/extensions/v2/package-builder"; + +const secretsFile = ".dev.vars.packages-local"; +const state = ".wrangler/package-demo"; +const base = "http://localhost:8789/extensions/v2"; +const action = process.argv[2]; +const fresh = process.argv.includes("--fresh"); +if (action === "setup") { + try { + await writeFile( + secretsFile, + `ASSERTION_SIGNING_SECRET="${Array.from(crypto.getRandomValues(new Uint8Array(32)), (b) => b.toString(16).padStart(2, "0")).join("")}"\nGITHUB_TOKEN=""\nEXTENSIONS_REVALIDATE_SECRET=""\n`, + { flag: "wx", mode: 0o600 } + ); + } catch (error) { + if (!(error instanceof Error && "code" in error && error.code === "EEXIST")) + throw error; + } + function wrangler(args: string[]) { + execFileSync( + process.execPath, + [ + "node_modules/wrangler/bin/wrangler.js", + ...args, + "--env", + "packages-local", + "--local", + "--persist-to", + state + ], + { + stdio: ["pipe", "pipe", "inherit"], + maxBuffer: 16 * 1024 * 1024, + env: { ...process.env, CI: "true" } + } + ); + } + for (const binding of ["DB_EXTENSIONS", "DB_CENTRAL_ALERTS"]) + wrangler(["d1", "migrations", "apply", binding]); + wrangler([ + "d1", + "execute", + "DB_EXTENSIONS", + "--command", + ` + INSERT INTO users (id,name,created_at,updated_at,is_moderator) VALUES + ('package-demo-owner','Local Package Developer',CURRENT_TIMESTAMP,CURRENT_TIMESTAMP,0), + ('package-demo-reviewer','Local Package Reviewer',CURRENT_TIMESTAMP,CURRENT_TIMESTAMP,1) + ON CONFLICT DO NOTHING; + INSERT INTO developers (id,type,name,owner_user_id,approved_at,created_at,updated_at) + VALUES ('package-demo-studio','user','Local Package Developer','package-demo-owner',CURRENT_TIMESTAMP,CURRENT_TIMESTAMP,CURRENT_TIMESTAMP) + ON CONFLICT DO NOTHING; + ` + ]); + console.info( + "Local package demo initialized. Start npm run packages:dev, then run npm run packages:demo." + ); +} else if (action === "exercise") { + const secret = (await readFile(secretsFile, "utf8")).match( + /^ASSERTION_SIGNING_SECRET="([a-f0-9]+)"$/m + )?.[1]; + if (!secret) throw new Error("Run npm run packages:setup first"); + async function request( + path: string, + init: RequestInit = {}, + user?: string + ): Promise { + const headers = new Headers(init.headers); + if (user) { + const now = Math.floor(Date.now() / 1000); + headers.set( + "Authorization", + `Bearer ${await sign({ sub: user, iat: now, exp: now + 60, iss: "fossbilling-extensions", aud: "fossbilling-api/extensions-v2", purpose: "user-authentication", ver: 1 }, secret!, "HS256")}` + ); + } + const response = await fetch(`${base}${path}`, { ...init, headers }); + if (path !== "/packages" || response.status !== 202) return response; + assert.ok(response.headers.get("Location")?.includes("/package-jobs/")); + let job = ( + (await response.json()) as { + result: { + id: string; + status: string; + artifact: unknown; + error: { status: number; message: string } | null; + }; + } + ).result; + assert.equal((await request(`/package-jobs/${job.id}`)).status, 401); + for (let attempt = 0; attempt < 120; attempt++) { + if (job.status === "ready") + return Response.json({ result: job.artifact }, { status: 201 }); + if (job.status === "failed") + return Response.json( + { error: job.error }, + { status: job.error?.status ?? 422 } + ); + await new Promise((resolve) => setTimeout(resolve, 1000)); + const status = await request(`/package-jobs/${job.id}`, {}, user); + assert.equal(status.status, 200, await status.clone().text()); + job = ((await status.json()) as { result: typeof job }).result; + } + throw new Error(`Package job ${job.id} did not complete locally`); + } + const manifest = { + manifest_version: 1, + type: "module", + id: fresh + ? `package-demo-${crypto.randomUUID().slice(0, 8)}` + : "directory-package-demo", + name: "Directory Package Demo", + version: "0.1.0", + description: "A locally built extension package", + author: { name: "Local Package Developer" }, + license: "MIT", + requires: { fossbilling: ">=0.8.0" } + }; + const writer = new ZipWriter(new Uint8ArrayWriter(), { + useWebWorkers: false, + useCompressionStream: true, + lastModDate: new Date("1980-01-01T00:00:00Z"), + extendedTimestamp: false + }); + await writer.add( + "Service.php", + new TextReader("' + ) + ); + const source = await writer.close(); + const form = new FormData(); + form.set("manifest", JSON.stringify(manifest)); + form.set("source", new File([source], "source.zip")); + const uploaded = await request( + "/packages", + { method: "POST", body: form }, + "package-demo-owner" + ); + const body = (await uploaded.json()) as { result: unknown }; + assert.equal(uploaded.status, 201, JSON.stringify(body)); + const artifact = PackageArtifactSchema.parse(body.result); + const privateCheck = await request(`/packages/${artifact.id}/download`); + if (fresh) assert.equal(privateCheck.status, 404); + else assert.ok([404, 200].includes(privateCheck.status)); // 200 on a repeat. + await privateCheck.body?.cancel(); + const iconPath = `/packages/${artifact.id}/icon`; + if (fresh) assert.equal((await request(iconPath)).status, 404); + const preview = await request(iconPath, {}, "package-demo-owner"); + assert.equal(preview.status, 200); + assert.equal(preview.headers.get("Content-Type"), "image/svg+xml"); + assert.match(preview.headers.get("Cache-Control") ?? "", /private/); + assert.equal(preview.headers.get("X-Content-Type-Options"), "nosniff"); + assert.match(preview.headers.get("Content-Security-Policy") ?? "", /sandbox/); + await preview.body?.cancel(); + const { id: _id, ...metadata } = artifact.manifest; + const content = { + ...metadata, + readme: "# Directory Package Demo\n\nBuilt and reviewed locally.", + source: { + type: "custom", + repo: "https://example.invalid/local-package-demo" + }, + download_url: artifact.download_url, + releases: [ + { + tag: manifest.version, + date: "2026-11-05", + requires: manifest.requires, + artifact_id: artifact.id, + sha256: artifact.sha256, + download_url: artifact.download_url + } + ] + }; + const created = await request( + "/extensions", + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ id: manifest.id, ...content }) + }, + "package-demo-owner" + ); + let revisionId: string | undefined; + if (created.status === 201) + revisionId = ((await created.json()) as { result: { revision_id: string } }) + .result.revision_id; + else { + assert.equal(created.status, 409, await created.text()); + const existing = await request( + `/extensions/${manifest.id}`, + {}, + "package-demo-owner" + ); + const listing = (await existing.json()) as { + result: { + pending_revision?: { id: string; content: { download_url: string } }; + published?: { download_url: string }; + }; + }; + if ( + listing.result.pending_revision?.content.download_url === + artifact.download_url + ) + revisionId = listing.result.pending_revision.id; + else + assert.equal( + listing.result.published?.download_url, + artifact.download_url, + "Existing demo has unrelated changes; it was left untouched" + ); + } + if (revisionId) { + const approved = await request( + `/extensions/${manifest.id}/revisions/${revisionId}/approve`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: "{}" + }, + "package-demo-reviewer" + ); + assert.equal(approved.status, 200, await approved.text()); + } + const download = await request(`/packages/${artifact.id}/download`); + assert.equal(download.status, 200); + assert.equal( + await sha256(new Uint8Array(await download.arrayBuffer())), + artifact.sha256 + ); + const publicIcon = await request(iconPath); + assert.equal(publicIcon.status, 200); + assert.equal(publicIcon.headers.get("Content-Type"), "image/svg+xml"); + await publicIcon.body?.cancel(); + const correction = new FormData(); + correction.set( + "manifest", + JSON.stringify({ + ...manifest, + description: "A corrected manifest using the same source ZIP" + }) + ); + correction.set("source_id", artifact.source_id); + const corrected = await request( + "/packages", + { method: "POST", body: correction }, + "package-demo-owner" + ); + assert.equal(corrected.status, 201); + const next = PackageArtifactSchema.parse( + ((await corrected.json()) as { result: unknown }).result + ); + assert.equal(next.source_id, artifact.source_id); + assert.notEqual(next.sha256, artifact.sha256); + assert.equal((await request(`/packages/${next.id}/download`)).status, 404); + console.info( + JSON.stringify( + { + publishedPackage: artifact.download_url, + sha256: artifact.sha256, + unpublishedCorrection: next.id, + sourceId: artifact.source_id, + checks: + "Private queue status, private package and icon, approval, public checksum and icon, immutable correction" + }, + null, + 2 + ) + ); +} else { + throw new Error("Use npm run packages:setup or npm run packages:demo"); +} diff --git a/src/app/index.ts b/src/app/index.ts index e03fddd5..d0189fd1 100644 --- a/src/app/index.ts +++ b/src/app/index.ts @@ -1,3 +1,7 @@ +import { + consumePackageJobs, + recoverPackageJobs +} from "../services/extensions/v2/packages/cloudflare-queue"; import { getExtensionsDb } from "../lib/db"; import { maintainExtensionResources, @@ -75,12 +79,14 @@ app.all("/*", (c) => { }); export default { + queue: consumePackageJobs, fetch: app.fetch, request: app.request.bind(app), scheduled: async ( _event: ScheduledController, env: CloudflareBindings ): Promise => { + await recoverPackageJobs(env); const db = getExtensionsDb(env.DB_EXTENSIONS); await maintainExtensionResources(db, { mode: env.EXTENSIONS_RETENTION_MODE diff --git a/src/lib/adapters/cloudflare/index.ts b/src/lib/adapters/cloudflare/index.ts index 34a22cae..89f80c7b 100644 --- a/src/lib/adapters/cloudflare/index.ts +++ b/src/lib/adapters/cloudflare/index.ts @@ -1,3 +1,4 @@ +import { CloudflareQueueAdapter } from "./queue"; import { IPlatformBindings } from "../../interfaces"; import { CloudflareKVAdapter } from "./cache"; import { CloudflareEnvironmentAdapter } from "./environment"; @@ -18,6 +19,13 @@ export function createCloudflareBindings( let bindings = bindingsCache.get(env); if (!bindings) { bindings = { + queues: env.EXTENSION_PACKAGE_QUEUE + ? { + EXTENSION_PACKAGE_QUEUE: new CloudflareQueueAdapter( + env.EXTENSION_PACKAGE_QUEUE + ) + } + : {}, caches: { CACHE_KV: new CloudflareKVAdapter(env.CACHE_KV), AUTH_KV: new CloudflareKVAdapter(env.AUTH_KV) diff --git a/src/lib/adapters/cloudflare/queue.ts b/src/lib/adapters/cloudflare/queue.ts new file mode 100644 index 00000000..495d5aea --- /dev/null +++ b/src/lib/adapters/cloudflare/queue.ts @@ -0,0 +1,9 @@ +import type { IMessageQueue } from "../../interfaces"; + +export class CloudflareQueueAdapter implements IMessageQueue { + constructor(private readonly queue: Queue) {} + + async send(body: unknown): Promise { + await this.queue.send(body, { contentType: "json" }); + } +} diff --git a/src/lib/context.ts b/src/lib/context.ts index 1bed40fd..5210bb67 100644 --- a/src/lib/context.ts +++ b/src/lib/context.ts @@ -1,7 +1,8 @@ import { Context } from "hono"; -import { ICache, IPlatformBindings } from "./interfaces"; +import { ICache, IMessageQueue, IPlatformBindings } from "./interfaces"; export interface PlatformContext { + getQueue(name: string): IMessageQueue; getCache(name: string): ICache; getEnv(key: string): string | undefined; readonly raw: Context; @@ -12,6 +13,11 @@ export function createPlatformContext( bindings: IPlatformBindings ): PlatformContext { return { + getQueue(name: string): IMessageQueue { + const queue = bindings.queues?.[name]; + if (!queue) throw new Error(`Queue binding '${name}' not found`); + return queue; + }, getCache(name: string): ICache { const cache = bindings.caches[name]; if (!cache) { diff --git a/src/lib/interfaces.ts b/src/lib/interfaces.ts index dfdfc75d..a4523baa 100644 --- a/src/lib/interfaces.ts +++ b/src/lib/interfaces.ts @@ -21,7 +21,12 @@ export interface IEnvironment { get(key: string): string | undefined; } +export interface IMessageQueue { + send(body: unknown): Promise; +} + export interface IPlatformBindings { + queues?: Record; caches: Record; environment: IEnvironment; } diff --git a/src/lib/releases.ts b/src/lib/releases.ts index b3093057..6548f596 100644 --- a/src/lib/releases.ts +++ b/src/lib/releases.ts @@ -1,7 +1,9 @@ -import { rcompare, valid } from "semver"; +import { prerelease, rcompare, valid } from "semver"; export interface ReleaseTag { tag: string; + date?: string; + is_latest?: boolean; } // Newest first by semver. Tags that aren't valid semver can't be ordered @@ -22,3 +24,29 @@ export function sortReleasesDescending( ...withInvalidTag ]; } + +// Explicit selection wins; older records without a selection prefer stable versions. +export function latestRelease( + releases: readonly T[] +): T | undefined { + const selected = releases.find((release) => release.is_latest === true); + if (selected) return selected; + const stable = releases.filter( + (release) => valid(release.tag) && prerelease(release.tag) === null + ); + return (stable.length ? stable : releases).reduce( + (latest, release) => { + if (!latest) return release; + const a = valid(release.tag), + b = valid(latest.tag); + const order = a && b ? rcompare(a, b) : a ? -1 : b ? 1 : 0; + if (order) return order < 0 ? release : latest; + const time = (date?: string) => { + const value = Date.parse(date ?? ""); + return Number.isFinite(value) ? value : -Infinity; + }; + return time(release.date) > time(latest.date) ? release : latest; + }, + undefined + ); +} diff --git a/src/services/extensions/v1/database.ts b/src/services/extensions/v1/database.ts index 95a4d8b2..f4e41730 100644 --- a/src/services/extensions/v1/database.ts +++ b/src/services/extensions/v1/database.ts @@ -3,7 +3,7 @@ import { and, eq, isNotNull, isNull, sql } from "drizzle-orm"; import { ExtensionsDb } from "../../../lib/db"; import { extensions, developers } from "../v2/db/schema"; import { DatabaseResult } from "../../../lib/interfaces"; -import { sortReleasesDescending } from "../../../lib/releases"; +import { latestRelease, sortReleasesDescending } from "../../../lib/releases"; import { parseJSON } from "../../../lib/json"; import type { Release as ManifestRelease } from "../v2/schemas/extensions"; import { Extension, Release, Author, Repository } from "./interfaces"; @@ -187,10 +187,10 @@ export class ExtensionsDatabase { ) ); if (!row) return this.notFound(id); - const releases = sortReleasesDescending( + const release = latestRelease( JSON.parse(row.releases!) as ManifestRelease[] ); - return { data: { version: releases[0]?.tag ?? null }, error: null }; + return { data: { version: release?.tag ?? null }, error: null }; } catch (error) { return this.databaseError(error); } diff --git a/src/services/extensions/v1/interfaces.ts b/src/services/extensions/v1/interfaces.ts index 79429486..1bbca993 100644 --- a/src/services/extensions/v1/interfaces.ts +++ b/src/services/extensions/v1/interfaces.ts @@ -54,6 +54,7 @@ export type User = { export type Release = { tag: string; + is_latest?: boolean; date: string; download_url: string; changelog_url?: string; diff --git a/src/services/extensions/v2/PACKAGE-RUNTIME.md b/src/services/extensions/v2/PACKAGE-RUNTIME.md new file mode 100644 index 00000000..106ccc70 --- /dev/null +++ b/src/services/extensions/v2/PACKAGE-RUNTIME.md @@ -0,0 +1,38 @@ +# Package runtime validation + +Package preparation accepts ready-to-install ZIPs. The directory never runs +uploaded code, compilers, dependency installation or package build scripts. + +The HTTP endpoint validates the bounded request, stores the upload privately and +returns a job ID. The queue consumer validates ZIP entries, copies their existing +compressed payloads, replaces the manifest and prepares the display icon. + +## Local checks + +The standalone demo uses the real Wrangler queue consumer with local D1 and R2: + +```sh +npm run packages:setup +npm run packages:dev +# In a second terminal: +npm run packages:demo -- --fresh +``` + +A fresh run verifies private job status, package and icon access before approval, +owner preview security headers, review and approval, the public ZIP checksum and +icon, and a private manifest correction using the stored source. + +Worker and Node tests additionally cover archive limits and corruption, queue +retries and duplicate delivery, interrupted leases, ownership changes and +publisher revocation. The directory tests cover the private job proxy and polling +failures; publisher tests cover completion, rejection and the polling deadline. + +## Before production enablement + +Measure admission and consumer invocations separately: CPU and wall duration, +memory failures, queue delay, retries, and storage/image-service latency. Include +near-limit ZIPs and SVG/raster icons. The consumer is configured for one message +at a time with concurrency one. + +Local functional tests and earlier process-CPU benchmarks do not establish +production capacity or billing. diff --git a/src/services/extensions/v2/PACKAGING.md b/src/services/extensions/v2/PACKAGING.md new file mode 100644 index 00000000..05050157 --- /dev/null +++ b/src/services/extensions/v2/PACKAGING.md @@ -0,0 +1,260 @@ +# Directory-built packages + +Authors upload ready-to-install distribution ZIPs, with any compilation already +complete. The API validates and repacks them with directory-generated manifests, +then publishes them through the existing revision review. It never runs uploaded +code or package build scripts. Package storage uses the private B2 bucket when configured, otherwise Wrangler's local R2 emulator. Pending images continue to use the private image binding. +Package routes are disabled by default. Local development and production enablement are separate modes. + +## Run the local demo + +Use Node.js 24, install dependencies with `npm ci`, then: + +```sh +npm run packages:setup +npm run packages:dev +``` + +In another terminal: + +```sh +npm run packages:demo -- --fresh +``` + +Setup creates an ignored `.dev.vars.packages-local`, applies local migrations, +and seeds a developer and reviewer. Data lives in `.wrangler/package-demo`; +the API listens on port 8789. The demo uploads a synthetic module, checks private +access, submits and approves it, verifies the public download checksum, and +builds a manifest correction from the stored source. `--fresh` allocates a new +extension ID and includes an SVG icon, so private access is checked before any +publication. It also verifies anonymous job-status denial, owner preview headers, +and public icon access after approval. Without `--fresh`, the original no-icon +fixture is repeatable and reuses artifacts. + +## API and review + +The OpenAPI reference is available at `/extensions/v2/docs`. + +- `POST /extensions/v2/packages` accepts manifest v1 JSON and either a ready-to-install + ZIP or an existing `source_id` in a multipart form. It stores the upload privately + and returns **202** with a job ID, status and private status URL in `Location`. + An active owning developer profile is required. Machine publishers need + `extensions:push` and an active mapped developer binding. Moderators may + rebuild published stored source for a correction, but cannot upload new files + for another developer. +- `GET /extensions/v2/package-jobs/{jobId}` returns `queued`, `processing`, `ready` + or `failed`. Ready jobs include the artifact; failed jobs include an actionable + error. Reads require the current owning developer/publisher or a moderator, + and are never publicly cached. Identical requests reuse queued/completed work. +- `GET /extensions/v2/packages/{packageId}` returns the artifact metadata. + `/download` serves its ZIP and `/icon` serves its prepared display image. + Before approval, only the current owner and moderators have access. Delisting + removes public access. +- `POST /extensions/v2/packages/cleanup` lets moderators reclaim unreferenced + object reservations older than one day. + +Releases carry `artifact_id`, `sha256`, the download URL and manifest requirements. +Submission and approval validate these against the stored artifact. The selected +latest release also supplies the overview's version, download, requirements and +manifest-backed metadata. Editing a release requires its stored package or a +source ZIP; the directory always queues package preparation before saving it. + +At most one release can have `is_latest: true`. Without a selection, the newest +stable release is used, falling back to a prerelease. The directory's release +editor provides a “Set as Latest Release” checkbox, checked by default for new +stable releases. This flag stays out of the manifest. + +## Queue processing and provider boundaries + +Queue messages contain only job IDs. D1 stores the validated manifest, captured +owner/profile generation/ownership epoch, upload reference and processing result. +The consumer rechecks account ownership and machine-publisher revocation before +processing. The staged upload becomes the immutable source object after validation, +so it is not uploaded to package storage a second time. + +The generic `IMessageQueue` producer interface lives in `src/lib/interfaces.ts`; +`CloudflareQueueAdapter` implements it. `processPackageJob` depends on a +`PackageJobRepository` and a preparation callback, not Cloudflare MessageBatch, +acknowledgements or retry APIs. The D1 repository and Cloudflare consumer wire the +existing storage/image implementation into that lifecycle. Node tests exercise +processing through in-memory implementations without a Cloudflare runtime. +This keeps the new job lifecycle portable; existing D1 and Images integrations +would still need their own adapters for a complete provider migration. + +Production and local configurations each declare a producer/consumer pair. The +consumer handles one message at a time, with concurrency limited to one. Duplicate +delivery uses a 20-minute processing lease; a busy duplicate is delayed until that +lease can expire. Temporary failures retry after a minute, at most three processing +attempts. Invalid archives and revoked ownership fail immediately. The normal +scheduled handler redispatches unsent or stale jobs; this covers interruption +between saving a job and publishing its queue message. It does not rely on the +browser or the HTTP request remaining connected. + +Jobs interrupted or undelivered for a day are marked expired once no active lease +remains. Terminal jobs are retained for seven days, with at most 256 jobs per owning +account. Cleanup never deletes uploads referenced by active jobs, and detaches +terminal job references before reclaiming abandoned objects. Published artifacts +remain immutable and are not removed with job history. + +The directory polls private job status and retains a release-job reference across +reloads. The publisher waits for completion before submitting a revision; failed +or timed-out jobs stop publication, and a retry reuses existing work. Server-side +new submissions and manifest corrections also poll with a bounded wait. + +## Storage and rebuilds + +Each developer profile generation, extension ID and version has one immutable +source ZIP. Changing any source bytes, including repacking, requires a new +version. Manifest changes produce a new immutable artifact from that source; +directory-only changes such as README edits keep the existing package. +Approval publishes the revision and its artifacts in one D1 transaction. Older +approved artifacts retain their URLs and checksums. + +Object reservations precede uploads. Unique constraints deduplicate concurrent +builds; failed writes remain recorded for cleanup. Cleanup claims unreferenced +objects before deletion, and builds cannot reference claimed objects. The +per-account allowance is 256 objects / 512 MiB, including abandoned reservations +and prepared images. Ready sources and artifacts are retained. + +ZIPs must contain ready-to-install files at the root. Packaging replaces the +optional root `manifest.json`. Other files are decompressed for CRC and actual +size validation, then their original compressed payloads are copied unchanged +with normalized ZIP headers. Only the generated manifest is compressed. It never runs +scripts from the archive. Input is limited to 5 MiB, 500 entries and 16 MiB of +actual decompressed data; multipart requests are capped at 6 MiB before parsing. +Encrypted entries, symlinks, unsafe paths, duplicate/colliding names, file/directory +conflicts and corrupt archives are rejected. Checksums cover the final stored ZIP. + +## Icons + +Include at most one root `icon.svg`, `icon.png`, `icon.webp`, or `icon.avif`. +There is no manifest icon field; packages without an icon use the generic icon. +Packaging prepares display images once in `EXTENSION_IMAGE_BUCKET`, shared by +all artifacts using the same source. The original icon stays unchanged in the ZIP. + +SVGs are sanitized with Cloudflare svg-hush through the pinned `@zeropress/svg-hush` +Workers adapter and served directly. SVG input is capped at 2 MiB and sanitized +output at 4 MiB. Raster inputs use the local `PACKAGE_IMAGES` binding to produce +AVIF and PNG variants within 256 × 256 pixels, preserving proportions and +transparency without enlarging small images. Requests prefer SVG when present, +otherwise AVIF when explicitly accepted, with PNG fallback. They read stored +images directly and never reopen a ZIP. + +Pending previews use the private image route with authorization, private cache +headers, `nosniff`, and a sandboxed Content Security Policy. They are never copied +to public B2. After approval, a background publication job copies the selected +icon to B2, then updates the published listing to its direct CDN URL. A failed +copy retains the existing image route; subsequent published-image reads retry. +The revision ID is checked before changing the listing URL, so a slow copy +cannot overwrite a newer publication. + +## Private B2 package testing + +Set these in the integrated local API's ignored `.dev.vars`. The directory +worktree's root `.dev.vars` may hold the development credentials, but the API +process must receive them in its own configuration file. The standalone demo +uses `.dev.vars.packages-local`. + +```dotenv +EXTENSION_PACKAGES_S3_ENDPOINT="https://s3.eu-central-003.backblazeb2.com" +EXTENSION_PACKAGES_S3_REGION="eu-central-003" +EXTENSION_PACKAGES_S3_BUCKET="fb-extensions-packages" +EXTENSION_PACKAGES_S3_ACCESS_KEY_ID="your-keyID" +EXTENSION_PACKAGES_S3_SECRET_ACCESS_KEY="your-applicationKey" +EXTENSION_PACKAGES_S3_PREFIX="local" +``` + +Keep this bucket private. Use a separate development key restricted to the bucket +with read, write and delete-file permissions. No public URL or domain is required. +Use `EXTENSION_PACKAGES_MODE=local` only outside production. The committed production configuration remains disabled. +With no S3 endpoint, ZIPs use the local R2 binding. An incomplete S3 configuration +or upstream failure returns 503 rather than silently switching storage backends. + +Source and built ZIPs use `local/packages/`. Object IDs are allocated +once by D1 before upload. Uploads carry a signed SHA-256 payload and private cache +headers. Reads are bounded to 17 MiB and verify artifact bytes before returning a +ZIP through the existing permission-checked download route. B2 URLs and credentials +are never returned to visitors. Images still use the separate private image +binding, with only approved display images copied to the public asset bucket. + +Cleanup identifies the stored B2 version and deletes it explicitly. Deleting only +by name would leave billable versions behind. Failed deletion retains the D1 +reservation for retry. + +## B2 and Cloudflare testing + +Add these to the local API's ignored `.dev.vars` (use +`.dev.vars.packages-local` for the standalone API demo): + +```dotenv +EXTENSION_ASSETS_S3_ENDPOINT="https://s3.eu-central-003.backblazeb2.com" +EXTENSION_ASSETS_S3_REGION="eu-central-003" +EXTENSION_ASSETS_S3_BUCKET="fb-extensions-assets" +EXTENSION_ASSETS_S3_ACCESS_KEY_ID="your-keyID" +EXTENSION_ASSETS_S3_SECRET_ACCESS_KEY="your-applicationKey" +EXTENSION_ASSETS_PUBLIC_URL="https://extensions-assets.fossbilling.net" +EXTENSION_ASSETS_S3_PREFIX="local" +``` + +An unset endpoint disables B2 publication. Test images use `local/icons/`; +production can select a separate prefix. URLs include an immutable image identity, +and uploads set `Cache-Control: public, max-age=31536000, immutable`. Raster +identities include both variant hashes so AVIF and PNG can share a URL stem. +The API checks existing objects before uploading and caches successful checks in +KV. Directory pages use direct CDN images with AVIF/PNG `` fallback, +bypassing the directory image transformer. + +The bucket is public. The hostname-scoped Cloudflare response-header rule +`extension_assets_security_headers` adds these headers, including on cached +responses: + +```text +Content-Security-Policy: sandbox; default-src 'none'; style-src 'unsafe-inline'; img-src data: +X-Content-Type-Options: nosniff +``` + +Only approved assets enter this bucket. Delisting stops access through the API, +but does not revoke previously published public image URLs or cached copies. +Keep B2 daily caps and alerts configured for download and transaction charges. + +## Production configuration (prepared locally) + +The top-level Wrangler configuration names the production resources but keeps +`EXTENSION_PACKAGES_MODE=disabled`. No production resources or secrets are +created by a local build or type-generation command. + +Before deployment (including an automatic deployment after merge): + +1. Provision the `extension-packages` Queue and the private R2 bucket `fb-extensions-private-images` used by + `EXTENSION_IMAGE_BUCKET`. Do not attach a public domain or enable public R2 + access. This stores prepared pending icons; ZIPs remain in private B2. + The bindings must exist even while package mode is disabled; that flag blocks + application routes, not deployment-time binding validation. +2. Supply `EXTENSION_PACKAGES_S3_ACCESS_KEY_ID` and + `EXTENSION_PACKAGES_S3_SECRET_ACCESS_KEY` as Worker secrets for + `fb-extensions-packages`. Supply separate `EXTENSION_ASSETS_S3_ACCESS_KEY_ID` + and `EXTENSION_ASSETS_S3_SECRET_ACCESS_KEY` for `fb-extensions-assets`. + Development keys in `.dev.vars` are not production secrets. +3. Confirm the Images binding is enabled for the account, the scoped CDN security + headers remain configured, and B2 caps/alerts are in place. +4. Apply the package and job tables through the normal D1 deployment workflow. +5. After queue-consumer capacity and moderation requirements are agreed, explicitly set + `EXTENSION_PACKAGES_MODE=enabled` in the reviewed production configuration. + Verify a production smoke test before enabling the dependent directory and + modules publisher changes. Those callers require an enabled, configured API; + merging a disabled API alone does not make package submission available. + +Production requires HTTPS download URLs with the `/extensions/v2` base path, +complete private B2 credentials, an explicit non-`local` object prefix, private +image storage and the Images binding. An incomplete configuration fails with 503; +it cannot fall back to local ZIP storage. Both B2 buckets use the `production` +prefix in this configuration. Existing local objects keep the `local` prefix. + +The disabled mode blocks package reads as well as writes. Once managed releases +are published, disabling it also stops their API downloads and private previews; +it is not an upload-only pause switch. Public CDN images already published remain +accessible independently. + +Local validation and the remaining production measurements are summarized in +[PACKAGE-RUNTIME.md](PACKAGE-RUNTIME.md). Local queue success does not establish +production CPU, memory or latency limits. diff --git a/src/services/extensions/v2/README.md b/src/services/extensions/v2/README.md index d5f36f4f..90b520d4 100644 --- a/src/services/extensions/v2/README.md +++ b/src/services/extensions/v2/README.md @@ -17,9 +17,9 @@ Endpoints are not listed here. The service publishes its own contract: New v2 writes use the [FOSSBilling manifest v1 fields](https://github.com/FOSSBilling/modules/blob/main/manifest.schema.json): `manifest_version: 1`, `type: "module"` (replacing `mod`; standalone `hook` entries require manual reclassification), `author: {name, url?}`, a string `license`, optional `homepage_url` (replacing `website`), and `requires: {fossbilling, php, modules?}`. IDs allow lowercase letters, digits, underscores, and hyphens, up to 64 characters. Names are limited to 100 characters, descriptions to 500, and versions use SemVer without a `v` prefix. -`icon_url` is an optional path relative to the package root, as in `icon.svg`; it cannot be an absolute URL, absolute path, or contain `..` or backslashes. Consumers resolve package assets against their trusted distribution source. Historical icons may still be absolute URLs. +Icons are discovered by convention: include one of `icon.svg`, `icon.png`, `icon.webp`, or `icon.avif` at the package root. The manifest and publishing input have no icon field. Public responses may include a computed `icon_url` for display; it is not editable. Directory packages without an icon return a generic fallback. -API publishing fields `readme`, `source`, `download_url`, and `releases` remain alongside the manifest metadata. Each release carries its own `requires` object instead of `min_fossbilling_version`. Constraints are retained verbatim, including Composer ranges and module dependencies; package CI remains responsible for semantic constraint/license validation and confirming that icon files exist. The API adds size bounds to metadata (author names: 200 characters; license and requirement strings: 1,000; URLs and icon paths: 2,048). +API publishing fields `readme`, `source`, `download_url`, and `releases` remain alongside the manifest metadata. Each release carries its own `requires` object instead of `min_fossbilling_version`. Constraints are retained verbatim, including Composer ranges and module dependencies; package CI remains responsible for semantic constraint/license validation. The API adds size bounds to metadata (author names: 200 characters; license and requirement strings: 1,000; URLs: 2,048). `requires.fossbilling` is mandatory. `requires.php` is an optional additional restriction: when omitted, FOSSBilling’s PHP requirements apply; when supplied, both the core and extension constraints must be satisfied. @@ -48,7 +48,7 @@ After applying the prepared migration, run the read-only `db/resource-inventory. Apply the migration and API deployment in a controlled cutover: the old Worker expects object licenses, while the new Worker requires the added metadata column. -This changes the v2 request/response contract: regenerate the Extensions site's client and update CI publishers together with the API rollout. The modules publisher must stop translating `module` to `mod` and send the manifest fields directly. Package-relative icons use v1's empty-string fallback. The v1 adapter continues to return `mod`, `website`, object licenses, and minimum release versions; constraints are converted to their earliest matching SemVer version. A malformed, unsatisfiable, or Composer-only constraint with no representable v1 minimum produces an explicit error identifying the release for manual correction; it is never replaced with `0.0.0`. +This changes the v2 request/response contract: regenerate the Extensions site's client and update CI publishers together with the API rollout. The modules publisher must stop translating `module` to `mod` and send the manifest fields directly. Icons follow the package filename convention. The v1 adapter continues to return `mod`, `website`, object licenses, and minimum release versions; constraints are converted to their earliest matching SemVer version. A malformed, unsatisfiable, or Composer-only constraint with no representable v1 minimum produces an explicit error identifying the release for manual correction; it is never replaced with `0.0.0`. The frontend revalidation tags are `extensions` and `developers`, shared with the Extensions site. Deploy the API and site tag change together. New code calls `revalidateExtensions(c)`. @@ -126,7 +126,7 @@ Assertions use HS256 with issuer `fossbilling-extensions`, audience `fossbilling Machine clients (CI publishers) authenticate with short-lived access tokens from the central auth service's `client_credentials` grant, bound to this API via RFC 8707 `resource=https://api.fossbilling.net`. Tokens are RFC 9068 JWTs (`typ: at+jwt`, EdDSA) carrying `client_id` and `scope`; verification is local via the issuer's JWKS (`M2M_*` variables, root README), so revocation takes effect at token expiry (10 minutes). -Two scopes gate the machine surface: `extensions:push` covers `POST /extensions` and `PUT /extensions/{id}`; `extensions:publish` additionally covers approving a pending revision on the mapped developer's own extension — the one carve-out from moderator-only approval. Machine tokens are answered 403 on account, moderator, and ownership routes; on public reads they are served the anonymous projection. Bindings are bound to the developer's ownership epoch at registration, so an ownership transfer goes stale and a moderator must re-register before the client writes again. +Two scopes gate the machine surface: `extensions:push` covers `POST /extensions`, `PUT /extensions/{id}`, and package builds through `POST /packages`; `extensions:publish` additionally covers approving a pending revision on the mapped developer's own extension — the one carve-out from moderator-only approval. Machine tokens are answered 403 on account, moderator, and ownership routes; on public reads they are served the anonymous projection. Bindings are bound to the developer's ownership epoch at registration, so an ownership transfer goes stale and a moderator must re-register before the client writes again. A moderator binds a client to a developer profile with `PUT /m2m/clients/{client_id}` (`developer_id`, `can_publish`), lists bindings with `GET /m2m/clients`, and revokes with `DELETE /m2m/clients/{client_id}`. A revoked client's tokens still verify but fail authorization on the next write. Each binding also creates a synthetic internal user (`m2m:`) that anchors revision foreign keys; it is not an account and cannot sign in. @@ -203,3 +203,9 @@ Reconcile and re-run; the migration touches nothing before these checks. ## Code layout See `AGENTS.md` for `routes/`, `db/`, `schemas/`, `github/`, and `middleware.ts`. This service is the reference layout for larger services. + +## Directory-built packages (local development) + +See [PACKAGING.md](./PACKAGING.md) for the authenticated upload/build API, +immutable artifacts, moderation integration, local test-data setup and storage +limits. Production package routes remain disabled until storage is configured. diff --git a/src/services/extensions/v2/db/badges.ts b/src/services/extensions/v2/db/badges.ts index 7bb151a0..3f066cd6 100644 --- a/src/services/extensions/v2/db/badges.ts +++ b/src/services/extensions/v2/db/badges.ts @@ -2,7 +2,7 @@ import { ZodError } from "zod"; import { and, isNotNull, isNull, sql } from "drizzle-orm"; import { ExtensionsDb } from "../../../../lib/db"; import { DatabaseResult } from "../../../../lib/interfaces"; -import { sortReleasesDescending } from "../../../../lib/releases"; +import { latestRelease } from "../../../../lib/releases"; import { BadgeDataSchema } from "../schemas/badges"; import { MAX_CONTENT_BYTES } from "../resource-limits"; import { databaseError } from "./errors"; @@ -51,7 +51,7 @@ export class ExtensionBadgesDatabase { }); return { data: { - latestRelease: sortReleasesDescending(data.releases)[0] ?? null, + latestRelease: latestRelease(data.releases) ?? null, license: data.license }, error: null diff --git a/src/services/extensions/v2/db/extensions.ts b/src/services/extensions/v2/db/extensions.ts index feadbb55..2bcfef70 100644 --- a/src/services/extensions/v2/db/extensions.ts +++ b/src/services/extensions/v2/db/extensions.ts @@ -1,3 +1,5 @@ +import { packageIconUrl } from "./packages"; +import { validatePackageContent } from "./packages"; import { MAX_CONTENT_BYTES } from "../resource-limits"; import { and, asc, eq, isNotNull, isNull, or, sql } from "drizzle-orm"; import { alias } from "drizzle-orm/sqlite-core"; @@ -636,6 +638,13 @@ export class ExtensionsDatabase { let results; try { + const packageError = await validatePackageContent( + this.db.$client, + input.extensionId, + input.developerId, + input.content + ); + if (packageError) return { data: null, error: packageError }; const extensionStmt = toD1Statement(this.db.$client, { sql: `INSERT INTO extensions (id, developer_id, created_by, created_at, updated_at) SELECT ?, d.id, d.owner_user_id, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP @@ -728,6 +737,13 @@ export class ExtensionsDatabase { let results; try { + const packageError = await validatePackageContent( + this.db.$client, + input.extensionId, + input.developerId, + input.content + ); + if (packageError) return { data: null, error: packageError }; const extensionStmt = toD1Statement(this.db.$client, { sql: `INSERT INTO extensions (id, developer_id, created_by, created_at, updated_at) SELECT ?, d.id, d.owner_user_id, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP @@ -1155,6 +1171,13 @@ export class ExtensionsDatabase { let results; try { + const packageError = await validatePackageContent( + this.db.$client, + id, + undefined, + valid + ); + if (packageError) return { data: null, error: packageError }; const correctStmt = toD1Statement(this.db.$client, { sql: `INSERT INTO extension_revisions (id, extension_id, developer_id, submitted_by, status, content, @@ -1205,7 +1228,7 @@ export class ExtensionsDatabase { JSON.stringify(valid.releases), valid.homepage_url ?? "", JSON.stringify(valid.license), - valid.icon_url ?? null, + packageIconUrl(valid), valid.readme, JSON.stringify(valid.source), valid.version, diff --git a/src/services/extensions/v2/db/migrations/0030_directory_packages.sql b/src/services/extensions/v2/db/migrations/0030_directory_packages.sql new file mode 100644 index 00000000..33cc3ca7 --- /dev/null +++ b/src/services/extensions/v2/db/migrations/0030_directory_packages.sql @@ -0,0 +1,82 @@ +CREATE TABLE `package_artifacts` ( + `id` text PRIMARY KEY NOT NULL, + `source_id` text NOT NULL, + `manifest` text NOT NULL, + `manifest_hash` text NOT NULL, + `sha256` text NOT NULL, + `object_id` text NOT NULL, + `download_url` text NOT NULL, + `published_at` text, + `created_at` text DEFAULT CURRENT_TIMESTAMP NOT NULL, + FOREIGN KEY (`source_id`) REFERENCES `package_sources`(`id`) ON UPDATE no action ON DELETE no action, + FOREIGN KEY (`object_id`) REFERENCES `package_objects`(`id`) ON UPDATE no action ON DELETE no action +); +--> statement-breakpoint +CREATE UNIQUE INDEX `idx_package_artifact_manifest` ON `package_artifacts` (`source_id`,`manifest_hash`);--> statement-breakpoint +CREATE TABLE `package_objects` ( + `deleting` integer DEFAULT 0 NOT NULL, + `id` text PRIMARY KEY NOT NULL, + `created_by` text NOT NULL, + `bytes` integer NOT NULL, + `created_at` text DEFAULT CURRENT_TIMESTAMP NOT NULL +); +--> statement-breakpoint +CREATE INDEX `idx_package_objects_creator` ON `package_objects` (`created_by`);--> statement-breakpoint +CREATE TABLE `package_sources` ( + `id` text PRIMARY KEY NOT NULL, + `profile_generation` text NOT NULL, + `extension_id` text NOT NULL, + `version` text NOT NULL, + `sha256` text NOT NULL, + `object_id` text NOT NULL, + `created_at` text DEFAULT CURRENT_TIMESTAMP NOT NULL, + FOREIGN KEY (`object_id`) REFERENCES `package_objects`(`id`) ON UPDATE no action ON DELETE no action +); +--> statement-breakpoint +CREATE UNIQUE INDEX `idx_package_source_version` ON `package_sources` (`profile_generation`,`extension_id`,`version`); +--> statement-breakpoint +CREATE TRIGGER package_sources_immutable BEFORE UPDATE ON package_sources +BEGIN SELECT RAISE(ABORT, 'package_source_immutable'); END; +--> statement-breakpoint +CREATE TRIGGER package_artifacts_immutable BEFORE UPDATE OF id,source_id,manifest,manifest_hash,sha256,object_id,download_url,created_at ON package_artifacts +BEGIN SELECT RAISE(ABORT, 'package_artifact_immutable'); END; +--> statement-breakpoint +CREATE TRIGGER package_artifacts_publication_immutable BEFORE UPDATE OF published_at ON package_artifacts WHEN OLD.published_at IS NOT NULL +BEGIN SELECT RAISE(ABORT, 'package_publication_immutable'); END; +--> statement-breakpoint +CREATE TRIGGER publish_package_artifacts AFTER UPDATE OF published_revision_id ON extensions +WHEN NEW.published_revision_id IS NOT NULL AND NEW.published_revision_id IS NOT OLD.published_revision_id +BEGIN + UPDATE package_artifacts SET published_at=CURRENT_TIMESTAMP + WHERE published_at IS NULL AND id IN ( + SELECT json_extract(value,'$.artifact_id') FROM json_each(NEW.releases) + ) AND source_id IN ( + SELECT s.id FROM package_sources s JOIN developers d ON d.profile_generation=s.profile_generation + WHERE lower(s.extension_id)=lower(NEW.id) AND d.id=NEW.developer_id + ); +END; + +--> statement-breakpoint +CREATE INDEX idx_package_sources_extension ON package_sources(extension_id); +--> statement-breakpoint +CREATE UNIQUE INDEX idx_package_sources_object ON package_sources(object_id); +--> statement-breakpoint +CREATE UNIQUE INDEX idx_package_artifacts_object ON package_artifacts(object_id); + +--> statement-breakpoint +CREATE TABLE `package_icons` ( + `source_id` text PRIMARY KEY NOT NULL, + `avif_object_id` text, + `avif_sha256` text, + `png_object_id` text, + `png_sha256` text, + `svg_object_id` text, + `svg_sha256` text, + FOREIGN KEY (`source_id`) REFERENCES `package_sources`(`id`) ON UPDATE no action ON DELETE no action, + FOREIGN KEY (`avif_object_id`) REFERENCES `package_objects`(`id`) ON UPDATE no action ON DELETE no action, + FOREIGN KEY (`png_object_id`) REFERENCES `package_objects`(`id`) ON UPDATE no action ON DELETE no action, + FOREIGN KEY (`svg_object_id`) REFERENCES `package_objects`(`id`) ON UPDATE no action ON DELETE no action +); +--> statement-breakpoint +CREATE TRIGGER package_icons_immutable BEFORE UPDATE ON package_icons +BEGIN SELECT RAISE(ABORT, 'Package icons are immutable'); END; diff --git a/src/services/extensions/v2/db/migrations/0031_package_jobs.sql b/src/services/extensions/v2/db/migrations/0031_package_jobs.sql new file mode 100644 index 00000000..70dd0f4f --- /dev/null +++ b/src/services/extensions/v2/db/migrations/0031_package_jobs.sql @@ -0,0 +1,28 @@ +CREATE TABLE `package_jobs` ( + `id` text PRIMARY KEY NOT NULL, + `fingerprint` text NOT NULL, + `owner_user_id` text NOT NULL, + `owner` text NOT NULL, + `manifest` text NOT NULL, + `source_id` text, + `upload_object_id` text, + `upload_sha256` text, + `status` text DEFAULT 'queued' NOT NULL, + `attempts` integer DEFAULT 0 NOT NULL, + `lease_id` text, + `lease_until` text, + `artifact_id` text, + `error_code` text, + `error_message` text, + `error_status` integer, + `enqueued_at` text, + `created_at` text DEFAULT CURRENT_TIMESTAMP NOT NULL, + `updated_at` text DEFAULT CURRENT_TIMESTAMP NOT NULL, + FOREIGN KEY (`upload_object_id`) REFERENCES `package_objects`(`id`) ON UPDATE no action ON DELETE no action, + FOREIGN KEY (`artifact_id`) REFERENCES `package_artifacts`(`id`) ON UPDATE no action ON DELETE no action, + CONSTRAINT "package_jobs_status_check" CHECK("package_jobs"."status" IN ('queued','processing','ready','failed')) +); +--> statement-breakpoint +CREATE UNIQUE INDEX `idx_package_jobs_fingerprint` ON `package_jobs` (`fingerprint`) WHERE "package_jobs"."status" != 'failed';--> statement-breakpoint +CREATE INDEX `idx_package_jobs_owner` ON `package_jobs` (`owner_user_id`);--> statement-breakpoint +CREATE INDEX `idx_package_jobs_dispatch` ON `package_jobs` (`status`,`enqueued_at`); \ No newline at end of file diff --git a/src/services/extensions/v2/db/migrations/meta/0030_snapshot.json b/src/services/extensions/v2/db/migrations/meta/0030_snapshot.json new file mode 100644 index 00000000..6b3fdc7b --- /dev/null +++ b/src/services/extensions/v2/db/migrations/meta/0030_snapshot.json @@ -0,0 +1,1708 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "3603892c-439a-4576-b6a3-95113bf62442", + "prevId": "522dcadb-834f-4249-9e4e-611e233a50d2", + "tables": { + "claim_verification_budgets": { + "name": "claim_verification_budgets", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_claim_verification_budgets_expiry": { + "name": "idx_claim_verification_budgets_expiry", + "columns": ["expires_at"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "developer_claims": { + "name": "developer_claims", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "developer_id": { + "name": "developer_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "claimant_id": { + "name": "claimant_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "note": { + "name": "note", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "review_note": { + "name": "review_note", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reviewer_id": { + "name": "reviewer_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + }, + "reviewed_at": { + "name": "reviewed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "github_org_verified": { + "name": "github_org_verified", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "github_verification_note": { + "name": "github_verification_note", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "idx_developer_claims_developer": { + "name": "idx_developer_claims_developer", + "columns": ["developer_id"], + "isUnique": false + }, + "idx_developer_claims_claimant": { + "name": "idx_developer_claims_claimant", + "columns": ["claimant_id"], + "isUnique": false + }, + "idx_developer_claims_pending_unique": { + "name": "idx_developer_claims_pending_unique", + "columns": ["developer_id", "claimant_id"], + "isUnique": true, + "where": "\"developer_claims\".\"status\" = 'pending'" + }, + "idx_developer_claims_pending_queue": { + "name": "idx_developer_claims_pending_queue", + "columns": ["created_at"], + "isUnique": false, + "where": "\"developer_claims\".\"status\" = 'pending'" + } + }, + "foreignKeys": { + "developer_claims_developer_id_developers_id_fk": { + "name": "developer_claims_developer_id_developers_id_fk", + "tableFrom": "developer_claims", + "tableTo": "developers", + "columnsFrom": ["developer_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "developer_claims_claimant_id_users_id_fk": { + "name": "developer_claims_claimant_id_users_id_fk", + "tableFrom": "developer_claims", + "tableTo": "users", + "columnsFrom": ["claimant_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "developer_claims_reviewer_id_users_id_fk": { + "name": "developer_claims_reviewer_id_users_id_fk", + "tableFrom": "developer_claims", + "tableTo": "users", + "columnsFrom": ["reviewer_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": { + "developer_claims_status_check": { + "name": "developer_claims_status_check", + "value": "\"developer_claims\".\"status\" IN ('pending', 'approved', 'rejected')" + }, + "developer_claims_github_org_verified_check": { + "name": "developer_claims_github_org_verified_check", + "value": "\"developer_claims\".\"github_org_verified\" IN (0, 1)" + } + } + }, + "developer_history": { + "name": "developer_history", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "developer_id": { + "name": "developer_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "changed_by": { + "name": "changed_by", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "changed_at": { + "name": "changed_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + } + }, + "indexes": { + "idx_developer_history_developer_changed_at": { + "name": "idx_developer_history_developer_changed_at", + "columns": ["developer_id", "changed_at"], + "isUnique": false + }, + "idx_developer_history_account_changed_at": { + "name": "idx_developer_history_account_changed_at", + "columns": ["changed_by", "changed_at"], + "isUnique": false + } + }, + "foreignKeys": { + "developer_history_changed_by_users_id_fk": { + "name": "developer_history_changed_by_users_id_fk", + "tableFrom": "developer_history", + "tableTo": "users", + "columnsFrom": ["changed_by"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "developer_transfers": { + "name": "developer_transfers", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "developer_id": { + "name": "developer_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token_hash": { + "name": "token_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + }, + "expires_at": { + "name": "expires_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "accepted_by": { + "name": "accepted_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "accepted_at": { + "name": "accepted_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "revoked_at": { + "name": "revoked_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "idx_developer_transfers_token": { + "name": "idx_developer_transfers_token", + "columns": ["token_hash"], + "isUnique": true + }, + "idx_developer_transfers_pending": { + "name": "idx_developer_transfers_pending", + "columns": ["developer_id"], + "isUnique": true, + "where": "\"developer_transfers\".\"accepted_at\" IS NULL AND \"developer_transfers\".\"revoked_at\" IS NULL" + } + }, + "foreignKeys": { + "developer_transfers_developer_id_developers_id_fk": { + "name": "developer_transfers_developer_id_developers_id_fk", + "tableFrom": "developer_transfers", + "tableTo": "developers", + "columnsFrom": ["developer_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "developer_transfers_created_by_users_id_fk": { + "name": "developer_transfers_created_by_users_id_fk", + "tableFrom": "developer_transfers", + "tableTo": "users", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "developer_transfers_accepted_by_users_id_fk": { + "name": "developer_transfers_accepted_by_users_id_fk", + "tableFrom": "developer_transfers", + "tableTo": "users", + "columnsFrom": ["accepted_by"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "developers": { + "name": "developers", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "owner_user_id": { + "name": "owner_user_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "approved_at": { + "name": "approved_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'1970-01-01T00:00:00.000Z'" + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'1970-01-01T00:00:00.000Z'" + }, + "avatar_url": { + "name": "avatar_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "contact_email": { + "name": "contact_email", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "ownership_epoch": { + "name": "ownership_epoch", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "content_revision": { + "name": "content_revision", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "profile_generation": { + "name": "profile_generation", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "approved_revision": { + "name": "approved_revision", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "approved_by": { + "name": "approved_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "github_org_verified": { + "name": "github_org_verified", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "github_verification_note": { + "name": "github_verification_note", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "github_verified_at": { + "name": "github_verified_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "github_url_verified": { + "name": "github_url_verified", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "url_check_cooldown_until": { + "name": "url_check_cooldown_until", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "idx_developers_owner_unique": { + "name": "idx_developers_owner_unique", + "columns": ["owner_user_id"], + "isUnique": true + }, + "idx_developers_approved": { + "name": "idx_developers_approved", + "columns": ["approved_at"], + "isUnique": false + } + }, + "foreignKeys": { + "developers_owner_user_id_users_id_fk": { + "name": "developers_owner_user_id_users_id_fk", + "tableFrom": "developers", + "tableTo": "users", + "columnsFrom": ["owner_user_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": { + "developers_ownership_epoch_check": { + "name": "developers_ownership_epoch_check", + "value": "\"developers\".\"ownership_epoch\" >= 1" + }, + "developers_content_revision_check": { + "name": "developers_content_revision_check", + "value": "\"developers\".\"content_revision\" >= 1" + }, + "developers_github_org_verified_check": { + "name": "developers_github_org_verified_check", + "value": "\"developers\".\"github_org_verified\" IN (0, 1)" + }, + "developers_github_url_verified_check": { + "name": "developers_github_url_verified_check", + "value": "\"developers\".\"github_url_verified\" = 1" + } + } + }, + "extension_resource_usage": { + "name": "extension_resource_usage", + "columns": { + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "subject": { + "name": "subject", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes": { + "name": "bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "extensions": { + "name": "extensions", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "revisions": { + "name": "revisions", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + } + }, + "indexes": { + "idx_extension_resource_usage_subject": { + "name": "idx_extension_resource_usage_subject", + "columns": ["scope", "subject"], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "extension_revisions": { + "name": "extension_revisions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "extension_id": { + "name": "extension_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "developer_id": { + "name": "developer_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "submitted_by": { + "name": "submitted_by", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content_bytes": { + "name": "content_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "summary_name": { + "name": "summary_name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "summary_version": { + "name": "summary_version", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "summary_description": { + "name": "summary_description", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "content_readable": { + "name": "content_readable", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "content_hash": { + "name": "content_hash", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "compacted_at": { + "name": "compacted_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reviewer_id": { + "name": "reviewer_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "review_note": { + "name": "review_note", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + }, + "reviewed_at": { + "name": "reviewed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "ownership_epoch": { + "name": "ownership_epoch", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + } + }, + "indexes": { + "idx_extension_revisions_retention": { + "name": "idx_extension_revisions_retention", + "columns": ["compacted_at", "reviewed_at", "id", "status"], + "isUnique": false + }, + "idx_extension_revisions_submitted_by": { + "name": "idx_extension_revisions_submitted_by", + "columns": ["submitted_by"], + "isUnique": false + }, + "idx_extension_revisions_developer": { + "name": "idx_extension_revisions_developer", + "columns": ["developer_id"], + "isUnique": false + }, + "idx_extension_revisions_pending": { + "name": "idx_extension_revisions_pending", + "columns": ["extension_id"], + "isUnique": true, + "where": "\"extension_revisions\".\"status\" = 'pending'" + }, + "idx_extension_revisions_extension_page": { + "name": "idx_extension_revisions_extension_page", + "columns": ["extension_id", "\"created_at\" desc", "\"id\" desc"], + "isUnique": false + }, + "idx_extension_revisions_submitter_page": { + "name": "idx_extension_revisions_submitter_page", + "columns": ["submitted_by", "\"created_at\" desc", "\"id\" desc"], + "isUnique": false + }, + "idx_extension_revisions_queue_page": { + "name": "idx_extension_revisions_queue_page", + "columns": ["status", "created_at", "id"], + "isUnique": false + }, + "idx_extension_revisions_reviewed": { + "name": "idx_extension_revisions_reviewed", + "columns": ["extension_id", "reviewed_at"], + "isUnique": false, + "where": "\"extension_revisions\".\"status\" IN ('approved', 'rejected')" + }, + "idx_extension_revisions_submitter_pending": { + "name": "idx_extension_revisions_submitter_pending", + "columns": ["submitted_by"], + "isUnique": false, + "where": "\"extension_revisions\".\"status\" = 'pending'" + } + }, + "foreignKeys": { + "extension_revisions_extension_id_extensions_id_fk": { + "name": "extension_revisions_extension_id_extensions_id_fk", + "tableFrom": "extension_revisions", + "tableTo": "extensions", + "columnsFrom": ["extension_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "extension_revisions_submitted_by_users_id_fk": { + "name": "extension_revisions_submitted_by_users_id_fk", + "tableFrom": "extension_revisions", + "tableTo": "users", + "columnsFrom": ["submitted_by"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "extension_revisions_reviewer_id_users_id_fk": { + "name": "extension_revisions_reviewer_id_users_id_fk", + "tableFrom": "extension_revisions", + "tableTo": "users", + "columnsFrom": ["reviewer_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": { + "extension_revisions_status_check": { + "name": "extension_revisions_status_check", + "value": "\"extension_revisions\".\"status\" IN ('pending', 'approved', 'rejected')" + }, + "extension_revisions_ownership_epoch_check": { + "name": "extension_revisions_ownership_epoch_check", + "value": "\"extension_revisions\".\"ownership_epoch\" >= 1" + } + } + }, + "extension_write_events": { + "name": "extension_write_events", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "developer_id": { + "name": "developer_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "occurred_at": { + "name": "occurred_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_extension_write_events_account": { + "name": "idx_extension_write_events_account", + "columns": ["account_id", "occurred_at"], + "isUnique": false + }, + "idx_extension_write_events_developer": { + "name": "idx_extension_write_events_developer", + "columns": ["developer_id", "occurred_at"], + "isUnique": false + }, + "idx_extension_write_events_time": { + "name": "idx_extension_write_events_time", + "columns": ["occurred_at"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "extensions": { + "name": "extensions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "developer_id": { + "name": "developer_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "published_bytes": { + "name": "published_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "published_at": { + "name": "published_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "published_revision_id": { + "name": "published_revision_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "manifest": { + "name": "manifest", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "releases": { + "name": "releases", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "website": { + "name": "website", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "license": { + "name": "license", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "icon_url": { + "name": "icon_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "readme": { + "name": "readme", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "version": { + "name": "version", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "download_url": { + "name": "download_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + }, + "delisted_at": { + "name": "delisted_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "delist_reason": { + "name": "delist_reason", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "idx_extensions_published_revision": { + "name": "idx_extensions_published_revision", + "columns": ["published_revision_id"], + "isUnique": false + }, + "idx_extensions_id_nocase": { + "name": "idx_extensions_id_nocase", + "columns": ["lower(\"id\")"], + "isUnique": true + }, + "idx_extensions_developer_order": { + "name": "idx_extensions_developer_order", + "columns": ["developer_id", "lower(\"id\")", "id"], + "isUnique": false + }, + "idx_extensions_public_order": { + "name": "idx_extensions_public_order", + "columns": ["lower(\"id\")", "id"], + "isUnique": false, + "where": "\"extensions\".\"published_at\" IS NOT NULL AND \"extensions\".\"delisted_at\" IS NULL" + }, + "idx_extensions_type_public_order": { + "name": "idx_extensions_type_public_order", + "columns": ["type", "lower(\"id\")", "id"], + "isUnique": false, + "where": "\"extensions\".\"published_at\" IS NOT NULL AND \"extensions\".\"delisted_at\" IS NULL" + } + }, + "foreignKeys": { + "extensions_developer_id_developers_id_fk": { + "name": "extensions_developer_id_developers_id_fk", + "tableFrom": "extensions", + "tableTo": "developers", + "columnsFrom": ["developer_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": { + "extensions_published_content_check": { + "name": "extensions_published_content_check", + "value": "\"extensions\".\"published_at\" IS NULL OR (\n \"extensions\".\"type\" IS NOT NULL AND \"extensions\".\"name\" IS NOT NULL AND\n \"extensions\".\"description\" IS NOT NULL AND \"extensions\".\"releases\" IS NOT NULL AND\n \"extensions\".\"website\" IS NOT NULL AND \"extensions\".\"license\" IS NOT NULL AND\n \"extensions\".\"readme\" IS NOT NULL AND \"extensions\".\"source\" IS NOT NULL AND\n \"extensions\".\"version\" IS NOT NULL AND \"extensions\".\"download_url\" IS NOT NULL\n )" + } + } + }, + "m2m_publisher_clients": { + "name": "m2m_publisher_clients", + "columns": { + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "developer_id": { + "name": "developer_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "can_publish": { + "name": "can_publish", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "ownership_epoch": { + "name": "ownership_epoch", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + }, + "revoked_at": { + "name": "revoked_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "idx_m2m_publisher_clients_developer": { + "name": "idx_m2m_publisher_clients_developer", + "columns": ["developer_id"], + "isUnique": false + } + }, + "foreignKeys": { + "m2m_publisher_clients_developer_id_developers_id_fk": { + "name": "m2m_publisher_clients_developer_id_developers_id_fk", + "tableFrom": "m2m_publisher_clients", + "tableTo": "developers", + "columnsFrom": ["developer_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": { + "m2m_publisher_clients_can_publish_check": { + "name": "m2m_publisher_clients_can_publish_check", + "value": "\"m2m_publisher_clients\".\"can_publish\" IN (0, 1)" + }, + "m2m_publisher_clients_ownership_epoch_check": { + "name": "m2m_publisher_clients_ownership_epoch_check", + "value": "\"m2m_publisher_clients\".\"ownership_epoch\" >= 1" + } + } + }, + "package_artifacts": { + "name": "package_artifacts", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "manifest": { + "name": "manifest", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "manifest_hash": { + "name": "manifest_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sha256": { + "name": "sha256", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "object_id": { + "name": "object_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "download_url": { + "name": "download_url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "published_at": { + "name": "published_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + } + }, + "indexes": { + "idx_package_artifact_manifest": { + "name": "idx_package_artifact_manifest", + "columns": ["source_id", "manifest_hash"], + "isUnique": true + }, + "idx_package_artifacts_object": { + "name": "idx_package_artifacts_object", + "columns": ["object_id"], + "isUnique": true + } + }, + "foreignKeys": { + "package_artifacts_source_id_package_sources_id_fk": { + "name": "package_artifacts_source_id_package_sources_id_fk", + "tableFrom": "package_artifacts", + "tableTo": "package_sources", + "columnsFrom": ["source_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "package_artifacts_object_id_package_objects_id_fk": { + "name": "package_artifacts_object_id_package_objects_id_fk", + "tableFrom": "package_artifacts", + "tableTo": "package_objects", + "columnsFrom": ["object_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "package_icons": { + "name": "package_icons", + "columns": { + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "avif_object_id": { + "name": "avif_object_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "avif_sha256": { + "name": "avif_sha256", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "png_object_id": { + "name": "png_object_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "png_sha256": { + "name": "png_sha256", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "svg_object_id": { + "name": "svg_object_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "svg_sha256": { + "name": "svg_sha256", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "package_icons_source_id_package_sources_id_fk": { + "name": "package_icons_source_id_package_sources_id_fk", + "tableFrom": "package_icons", + "tableTo": "package_sources", + "columnsFrom": ["source_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "package_icons_avif_object_id_package_objects_id_fk": { + "name": "package_icons_avif_object_id_package_objects_id_fk", + "tableFrom": "package_icons", + "tableTo": "package_objects", + "columnsFrom": ["avif_object_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "package_icons_png_object_id_package_objects_id_fk": { + "name": "package_icons_png_object_id_package_objects_id_fk", + "tableFrom": "package_icons", + "tableTo": "package_objects", + "columnsFrom": ["png_object_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "package_icons_svg_object_id_package_objects_id_fk": { + "name": "package_icons_svg_object_id_package_objects_id_fk", + "tableFrom": "package_icons", + "tableTo": "package_objects", + "columnsFrom": ["svg_object_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "package_objects": { + "name": "package_objects", + "columns": { + "deleting": { + "name": "deleting", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes": { + "name": "bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + } + }, + "indexes": { + "idx_package_objects_creator": { + "name": "idx_package_objects_creator", + "columns": ["created_by"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "package_sources": { + "name": "package_sources", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "profile_generation": { + "name": "profile_generation", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "extension_id": { + "name": "extension_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "version": { + "name": "version", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sha256": { + "name": "sha256", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "object_id": { + "name": "object_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + } + }, + "indexes": { + "idx_package_source_version": { + "name": "idx_package_source_version", + "columns": ["profile_generation", "extension_id", "version"], + "isUnique": true + }, + "idx_package_sources_extension": { + "name": "idx_package_sources_extension", + "columns": ["extension_id"], + "isUnique": false + }, + "idx_package_sources_object": { + "name": "idx_package_sources_object", + "columns": ["object_id"], + "isUnique": true + } + }, + "foreignKeys": { + "package_sources_object_id_package_objects_id_fk": { + "name": "package_sources_object_id_package_objects_id_fk", + "tableFrom": "package_sources", + "tableTo": "package_objects", + "columnsFrom": ["object_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "users": { + "name": "users", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "email_verified": { + "name": "email_verified", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "picture": { + "name": "picture", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "is_moderator": { + "name": "is_moderator", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "github_login": { + "name": "github_login", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "github_orgs": { + "name": "github_orgs", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "github_orgs_expires_at": { + "name": "github_orgs_expires_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": { + "idx_extension_revisions_extension_page": { + "columns": { + "\"created_at\" desc": { + "isExpression": true + }, + "\"id\" desc": { + "isExpression": true + } + } + }, + "idx_extension_revisions_submitter_page": { + "columns": { + "\"created_at\" desc": { + "isExpression": true + }, + "\"id\" desc": { + "isExpression": true + } + } + }, + "idx_extensions_id_nocase": { + "columns": { + "lower(\"id\")": { + "isExpression": true + } + } + }, + "idx_extensions_developer_order": { + "columns": { + "lower(\"id\")": { + "isExpression": true + } + } + }, + "idx_extensions_public_order": { + "columns": { + "lower(\"id\")": { + "isExpression": true + } + } + }, + "idx_extensions_type_public_order": { + "columns": { + "lower(\"id\")": { + "isExpression": true + } + } + } + } + } +} diff --git a/src/services/extensions/v2/db/migrations/meta/0031_snapshot.json b/src/services/extensions/v2/db/migrations/meta/0031_snapshot.json new file mode 100644 index 00000000..08d7f6a8 --- /dev/null +++ b/src/services/extensions/v2/db/migrations/meta/0031_snapshot.json @@ -0,0 +1,1896 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "67cf78d3-42a6-4777-a988-ebf2d8c4f9a9", + "prevId": "3603892c-439a-4576-b6a3-95113bf62442", + "tables": { + "claim_verification_budgets": { + "name": "claim_verification_budgets", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_claim_verification_budgets_expiry": { + "name": "idx_claim_verification_budgets_expiry", + "columns": ["expires_at"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "developer_claims": { + "name": "developer_claims", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "developer_id": { + "name": "developer_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "claimant_id": { + "name": "claimant_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "note": { + "name": "note", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "review_note": { + "name": "review_note", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reviewer_id": { + "name": "reviewer_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + }, + "reviewed_at": { + "name": "reviewed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "github_org_verified": { + "name": "github_org_verified", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "github_verification_note": { + "name": "github_verification_note", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "idx_developer_claims_developer": { + "name": "idx_developer_claims_developer", + "columns": ["developer_id"], + "isUnique": false + }, + "idx_developer_claims_claimant": { + "name": "idx_developer_claims_claimant", + "columns": ["claimant_id"], + "isUnique": false + }, + "idx_developer_claims_pending_unique": { + "name": "idx_developer_claims_pending_unique", + "columns": ["developer_id", "claimant_id"], + "isUnique": true, + "where": "\"developer_claims\".\"status\" = 'pending'" + }, + "idx_developer_claims_pending_queue": { + "name": "idx_developer_claims_pending_queue", + "columns": ["created_at"], + "isUnique": false, + "where": "\"developer_claims\".\"status\" = 'pending'" + } + }, + "foreignKeys": { + "developer_claims_developer_id_developers_id_fk": { + "name": "developer_claims_developer_id_developers_id_fk", + "tableFrom": "developer_claims", + "tableTo": "developers", + "columnsFrom": ["developer_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "developer_claims_claimant_id_users_id_fk": { + "name": "developer_claims_claimant_id_users_id_fk", + "tableFrom": "developer_claims", + "tableTo": "users", + "columnsFrom": ["claimant_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "developer_claims_reviewer_id_users_id_fk": { + "name": "developer_claims_reviewer_id_users_id_fk", + "tableFrom": "developer_claims", + "tableTo": "users", + "columnsFrom": ["reviewer_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": { + "developer_claims_status_check": { + "name": "developer_claims_status_check", + "value": "\"developer_claims\".\"status\" IN ('pending', 'approved', 'rejected')" + }, + "developer_claims_github_org_verified_check": { + "name": "developer_claims_github_org_verified_check", + "value": "\"developer_claims\".\"github_org_verified\" IN (0, 1)" + } + } + }, + "developer_history": { + "name": "developer_history", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "developer_id": { + "name": "developer_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "changed_by": { + "name": "changed_by", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "changed_at": { + "name": "changed_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + } + }, + "indexes": { + "idx_developer_history_developer_changed_at": { + "name": "idx_developer_history_developer_changed_at", + "columns": ["developer_id", "changed_at"], + "isUnique": false + }, + "idx_developer_history_account_changed_at": { + "name": "idx_developer_history_account_changed_at", + "columns": ["changed_by", "changed_at"], + "isUnique": false + } + }, + "foreignKeys": { + "developer_history_changed_by_users_id_fk": { + "name": "developer_history_changed_by_users_id_fk", + "tableFrom": "developer_history", + "tableTo": "users", + "columnsFrom": ["changed_by"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "developer_transfers": { + "name": "developer_transfers", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "developer_id": { + "name": "developer_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token_hash": { + "name": "token_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + }, + "expires_at": { + "name": "expires_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "accepted_by": { + "name": "accepted_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "accepted_at": { + "name": "accepted_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "revoked_at": { + "name": "revoked_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "idx_developer_transfers_token": { + "name": "idx_developer_transfers_token", + "columns": ["token_hash"], + "isUnique": true + }, + "idx_developer_transfers_pending": { + "name": "idx_developer_transfers_pending", + "columns": ["developer_id"], + "isUnique": true, + "where": "\"developer_transfers\".\"accepted_at\" IS NULL AND \"developer_transfers\".\"revoked_at\" IS NULL" + } + }, + "foreignKeys": { + "developer_transfers_developer_id_developers_id_fk": { + "name": "developer_transfers_developer_id_developers_id_fk", + "tableFrom": "developer_transfers", + "tableTo": "developers", + "columnsFrom": ["developer_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "developer_transfers_created_by_users_id_fk": { + "name": "developer_transfers_created_by_users_id_fk", + "tableFrom": "developer_transfers", + "tableTo": "users", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "developer_transfers_accepted_by_users_id_fk": { + "name": "developer_transfers_accepted_by_users_id_fk", + "tableFrom": "developer_transfers", + "tableTo": "users", + "columnsFrom": ["accepted_by"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "developers": { + "name": "developers", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "owner_user_id": { + "name": "owner_user_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "approved_at": { + "name": "approved_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'1970-01-01T00:00:00.000Z'" + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'1970-01-01T00:00:00.000Z'" + }, + "avatar_url": { + "name": "avatar_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "contact_email": { + "name": "contact_email", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "ownership_epoch": { + "name": "ownership_epoch", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "content_revision": { + "name": "content_revision", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "profile_generation": { + "name": "profile_generation", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "approved_revision": { + "name": "approved_revision", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "approved_by": { + "name": "approved_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "github_org_verified": { + "name": "github_org_verified", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "github_verification_note": { + "name": "github_verification_note", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "github_verified_at": { + "name": "github_verified_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "github_url_verified": { + "name": "github_url_verified", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "url_check_cooldown_until": { + "name": "url_check_cooldown_until", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "idx_developers_owner_unique": { + "name": "idx_developers_owner_unique", + "columns": ["owner_user_id"], + "isUnique": true + }, + "idx_developers_approved": { + "name": "idx_developers_approved", + "columns": ["approved_at"], + "isUnique": false + } + }, + "foreignKeys": { + "developers_owner_user_id_users_id_fk": { + "name": "developers_owner_user_id_users_id_fk", + "tableFrom": "developers", + "tableTo": "users", + "columnsFrom": ["owner_user_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": { + "developers_ownership_epoch_check": { + "name": "developers_ownership_epoch_check", + "value": "\"developers\".\"ownership_epoch\" >= 1" + }, + "developers_content_revision_check": { + "name": "developers_content_revision_check", + "value": "\"developers\".\"content_revision\" >= 1" + }, + "developers_github_org_verified_check": { + "name": "developers_github_org_verified_check", + "value": "\"developers\".\"github_org_verified\" IN (0, 1)" + }, + "developers_github_url_verified_check": { + "name": "developers_github_url_verified_check", + "value": "\"developers\".\"github_url_verified\" = 1" + } + } + }, + "extension_resource_usage": { + "name": "extension_resource_usage", + "columns": { + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "subject": { + "name": "subject", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes": { + "name": "bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "extensions": { + "name": "extensions", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "revisions": { + "name": "revisions", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + } + }, + "indexes": { + "idx_extension_resource_usage_subject": { + "name": "idx_extension_resource_usage_subject", + "columns": ["scope", "subject"], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "extension_revisions": { + "name": "extension_revisions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "extension_id": { + "name": "extension_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "developer_id": { + "name": "developer_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "submitted_by": { + "name": "submitted_by", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content_bytes": { + "name": "content_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "summary_name": { + "name": "summary_name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "summary_version": { + "name": "summary_version", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "summary_description": { + "name": "summary_description", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "content_readable": { + "name": "content_readable", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "content_hash": { + "name": "content_hash", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "compacted_at": { + "name": "compacted_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reviewer_id": { + "name": "reviewer_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "review_note": { + "name": "review_note", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + }, + "reviewed_at": { + "name": "reviewed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "ownership_epoch": { + "name": "ownership_epoch", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + } + }, + "indexes": { + "idx_extension_revisions_retention": { + "name": "idx_extension_revisions_retention", + "columns": ["compacted_at", "reviewed_at", "id", "status"], + "isUnique": false + }, + "idx_extension_revisions_submitted_by": { + "name": "idx_extension_revisions_submitted_by", + "columns": ["submitted_by"], + "isUnique": false + }, + "idx_extension_revisions_developer": { + "name": "idx_extension_revisions_developer", + "columns": ["developer_id"], + "isUnique": false + }, + "idx_extension_revisions_pending": { + "name": "idx_extension_revisions_pending", + "columns": ["extension_id"], + "isUnique": true, + "where": "\"extension_revisions\".\"status\" = 'pending'" + }, + "idx_extension_revisions_extension_page": { + "name": "idx_extension_revisions_extension_page", + "columns": ["extension_id", "\"created_at\" desc", "\"id\" desc"], + "isUnique": false + }, + "idx_extension_revisions_submitter_page": { + "name": "idx_extension_revisions_submitter_page", + "columns": ["submitted_by", "\"created_at\" desc", "\"id\" desc"], + "isUnique": false + }, + "idx_extension_revisions_queue_page": { + "name": "idx_extension_revisions_queue_page", + "columns": ["status", "created_at", "id"], + "isUnique": false + }, + "idx_extension_revisions_reviewed": { + "name": "idx_extension_revisions_reviewed", + "columns": ["extension_id", "reviewed_at"], + "isUnique": false, + "where": "\"extension_revisions\".\"status\" IN ('approved', 'rejected')" + }, + "idx_extension_revisions_submitter_pending": { + "name": "idx_extension_revisions_submitter_pending", + "columns": ["submitted_by"], + "isUnique": false, + "where": "\"extension_revisions\".\"status\" = 'pending'" + } + }, + "foreignKeys": { + "extension_revisions_extension_id_extensions_id_fk": { + "name": "extension_revisions_extension_id_extensions_id_fk", + "tableFrom": "extension_revisions", + "tableTo": "extensions", + "columnsFrom": ["extension_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "extension_revisions_submitted_by_users_id_fk": { + "name": "extension_revisions_submitted_by_users_id_fk", + "tableFrom": "extension_revisions", + "tableTo": "users", + "columnsFrom": ["submitted_by"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "extension_revisions_reviewer_id_users_id_fk": { + "name": "extension_revisions_reviewer_id_users_id_fk", + "tableFrom": "extension_revisions", + "tableTo": "users", + "columnsFrom": ["reviewer_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": { + "extension_revisions_status_check": { + "name": "extension_revisions_status_check", + "value": "\"extension_revisions\".\"status\" IN ('pending', 'approved', 'rejected')" + }, + "extension_revisions_ownership_epoch_check": { + "name": "extension_revisions_ownership_epoch_check", + "value": "\"extension_revisions\".\"ownership_epoch\" >= 1" + } + } + }, + "extension_write_events": { + "name": "extension_write_events", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "developer_id": { + "name": "developer_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "occurred_at": { + "name": "occurred_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_extension_write_events_account": { + "name": "idx_extension_write_events_account", + "columns": ["account_id", "occurred_at"], + "isUnique": false + }, + "idx_extension_write_events_developer": { + "name": "idx_extension_write_events_developer", + "columns": ["developer_id", "occurred_at"], + "isUnique": false + }, + "idx_extension_write_events_time": { + "name": "idx_extension_write_events_time", + "columns": ["occurred_at"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "extensions": { + "name": "extensions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "developer_id": { + "name": "developer_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "published_bytes": { + "name": "published_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "published_at": { + "name": "published_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "published_revision_id": { + "name": "published_revision_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "manifest": { + "name": "manifest", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "releases": { + "name": "releases", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "website": { + "name": "website", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "license": { + "name": "license", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "icon_url": { + "name": "icon_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "readme": { + "name": "readme", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "version": { + "name": "version", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "download_url": { + "name": "download_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + }, + "delisted_at": { + "name": "delisted_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "delist_reason": { + "name": "delist_reason", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "idx_extensions_published_revision": { + "name": "idx_extensions_published_revision", + "columns": ["published_revision_id"], + "isUnique": false + }, + "idx_extensions_id_nocase": { + "name": "idx_extensions_id_nocase", + "columns": ["lower(\"id\")"], + "isUnique": true + }, + "idx_extensions_developer_order": { + "name": "idx_extensions_developer_order", + "columns": ["developer_id", "lower(\"id\")", "id"], + "isUnique": false + }, + "idx_extensions_public_order": { + "name": "idx_extensions_public_order", + "columns": ["lower(\"id\")", "id"], + "isUnique": false, + "where": "\"extensions\".\"published_at\" IS NOT NULL AND \"extensions\".\"delisted_at\" IS NULL" + }, + "idx_extensions_type_public_order": { + "name": "idx_extensions_type_public_order", + "columns": ["type", "lower(\"id\")", "id"], + "isUnique": false, + "where": "\"extensions\".\"published_at\" IS NOT NULL AND \"extensions\".\"delisted_at\" IS NULL" + } + }, + "foreignKeys": { + "extensions_developer_id_developers_id_fk": { + "name": "extensions_developer_id_developers_id_fk", + "tableFrom": "extensions", + "tableTo": "developers", + "columnsFrom": ["developer_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": { + "extensions_published_content_check": { + "name": "extensions_published_content_check", + "value": "\"extensions\".\"published_at\" IS NULL OR (\n \"extensions\".\"type\" IS NOT NULL AND \"extensions\".\"name\" IS NOT NULL AND\n \"extensions\".\"description\" IS NOT NULL AND \"extensions\".\"releases\" IS NOT NULL AND\n \"extensions\".\"website\" IS NOT NULL AND \"extensions\".\"license\" IS NOT NULL AND\n \"extensions\".\"readme\" IS NOT NULL AND \"extensions\".\"source\" IS NOT NULL AND\n \"extensions\".\"version\" IS NOT NULL AND \"extensions\".\"download_url\" IS NOT NULL\n )" + } + } + }, + "m2m_publisher_clients": { + "name": "m2m_publisher_clients", + "columns": { + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "developer_id": { + "name": "developer_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "can_publish": { + "name": "can_publish", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "ownership_epoch": { + "name": "ownership_epoch", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + }, + "revoked_at": { + "name": "revoked_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "idx_m2m_publisher_clients_developer": { + "name": "idx_m2m_publisher_clients_developer", + "columns": ["developer_id"], + "isUnique": false + } + }, + "foreignKeys": { + "m2m_publisher_clients_developer_id_developers_id_fk": { + "name": "m2m_publisher_clients_developer_id_developers_id_fk", + "tableFrom": "m2m_publisher_clients", + "tableTo": "developers", + "columnsFrom": ["developer_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": { + "m2m_publisher_clients_can_publish_check": { + "name": "m2m_publisher_clients_can_publish_check", + "value": "\"m2m_publisher_clients\".\"can_publish\" IN (0, 1)" + }, + "m2m_publisher_clients_ownership_epoch_check": { + "name": "m2m_publisher_clients_ownership_epoch_check", + "value": "\"m2m_publisher_clients\".\"ownership_epoch\" >= 1" + } + } + }, + "package_artifacts": { + "name": "package_artifacts", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "manifest": { + "name": "manifest", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "manifest_hash": { + "name": "manifest_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sha256": { + "name": "sha256", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "object_id": { + "name": "object_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "download_url": { + "name": "download_url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "published_at": { + "name": "published_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + } + }, + "indexes": { + "idx_package_artifact_manifest": { + "name": "idx_package_artifact_manifest", + "columns": ["source_id", "manifest_hash"], + "isUnique": true + }, + "idx_package_artifacts_object": { + "name": "idx_package_artifacts_object", + "columns": ["object_id"], + "isUnique": true + } + }, + "foreignKeys": { + "package_artifacts_source_id_package_sources_id_fk": { + "name": "package_artifacts_source_id_package_sources_id_fk", + "tableFrom": "package_artifacts", + "tableTo": "package_sources", + "columnsFrom": ["source_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "package_artifacts_object_id_package_objects_id_fk": { + "name": "package_artifacts_object_id_package_objects_id_fk", + "tableFrom": "package_artifacts", + "tableTo": "package_objects", + "columnsFrom": ["object_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "package_icons": { + "name": "package_icons", + "columns": { + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "avif_object_id": { + "name": "avif_object_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "avif_sha256": { + "name": "avif_sha256", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "png_object_id": { + "name": "png_object_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "png_sha256": { + "name": "png_sha256", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "svg_object_id": { + "name": "svg_object_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "svg_sha256": { + "name": "svg_sha256", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "package_icons_source_id_package_sources_id_fk": { + "name": "package_icons_source_id_package_sources_id_fk", + "tableFrom": "package_icons", + "tableTo": "package_sources", + "columnsFrom": ["source_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "package_icons_avif_object_id_package_objects_id_fk": { + "name": "package_icons_avif_object_id_package_objects_id_fk", + "tableFrom": "package_icons", + "tableTo": "package_objects", + "columnsFrom": ["avif_object_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "package_icons_png_object_id_package_objects_id_fk": { + "name": "package_icons_png_object_id_package_objects_id_fk", + "tableFrom": "package_icons", + "tableTo": "package_objects", + "columnsFrom": ["png_object_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "package_icons_svg_object_id_package_objects_id_fk": { + "name": "package_icons_svg_object_id_package_objects_id_fk", + "tableFrom": "package_icons", + "tableTo": "package_objects", + "columnsFrom": ["svg_object_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "package_jobs": { + "name": "package_jobs", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "fingerprint": { + "name": "fingerprint", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "owner_user_id": { + "name": "owner_user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "owner": { + "name": "owner", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "manifest": { + "name": "manifest", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "upload_object_id": { + "name": "upload_object_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "upload_sha256": { + "name": "upload_sha256", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'queued'" + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "lease_id": { + "name": "lease_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "lease_until": { + "name": "lease_until", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "artifact_id": { + "name": "artifact_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error_code": { + "name": "error_code", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error_status": { + "name": "error_status", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "enqueued_at": { + "name": "enqueued_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + } + }, + "indexes": { + "idx_package_jobs_fingerprint": { + "name": "idx_package_jobs_fingerprint", + "columns": ["fingerprint"], + "isUnique": true, + "where": "\"package_jobs\".\"status\" != 'failed'" + }, + "idx_package_jobs_owner": { + "name": "idx_package_jobs_owner", + "columns": ["owner_user_id"], + "isUnique": false + }, + "idx_package_jobs_dispatch": { + "name": "idx_package_jobs_dispatch", + "columns": ["status", "enqueued_at"], + "isUnique": false + } + }, + "foreignKeys": { + "package_jobs_upload_object_id_package_objects_id_fk": { + "name": "package_jobs_upload_object_id_package_objects_id_fk", + "tableFrom": "package_jobs", + "tableTo": "package_objects", + "columnsFrom": ["upload_object_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "package_jobs_artifact_id_package_artifacts_id_fk": { + "name": "package_jobs_artifact_id_package_artifacts_id_fk", + "tableFrom": "package_jobs", + "tableTo": "package_artifacts", + "columnsFrom": ["artifact_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": { + "package_jobs_status_check": { + "name": "package_jobs_status_check", + "value": "\"package_jobs\".\"status\" IN ('queued','processing','ready','failed')" + } + } + }, + "package_objects": { + "name": "package_objects", + "columns": { + "deleting": { + "name": "deleting", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes": { + "name": "bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + } + }, + "indexes": { + "idx_package_objects_creator": { + "name": "idx_package_objects_creator", + "columns": ["created_by"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "package_sources": { + "name": "package_sources", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "profile_generation": { + "name": "profile_generation", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "extension_id": { + "name": "extension_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "version": { + "name": "version", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sha256": { + "name": "sha256", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "object_id": { + "name": "object_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + } + }, + "indexes": { + "idx_package_source_version": { + "name": "idx_package_source_version", + "columns": ["profile_generation", "extension_id", "version"], + "isUnique": true + }, + "idx_package_sources_extension": { + "name": "idx_package_sources_extension", + "columns": ["extension_id"], + "isUnique": false + }, + "idx_package_sources_object": { + "name": "idx_package_sources_object", + "columns": ["object_id"], + "isUnique": true + } + }, + "foreignKeys": { + "package_sources_object_id_package_objects_id_fk": { + "name": "package_sources_object_id_package_objects_id_fk", + "tableFrom": "package_sources", + "tableTo": "package_objects", + "columnsFrom": ["object_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "users": { + "name": "users", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "email_verified": { + "name": "email_verified", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "picture": { + "name": "picture", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "is_moderator": { + "name": "is_moderator", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "github_login": { + "name": "github_login", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "github_orgs": { + "name": "github_orgs", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "github_orgs_expires_at": { + "name": "github_orgs_expires_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": { + "idx_extension_revisions_extension_page": { + "columns": { + "\"created_at\" desc": { + "isExpression": true + }, + "\"id\" desc": { + "isExpression": true + } + } + }, + "idx_extension_revisions_submitter_page": { + "columns": { + "\"created_at\" desc": { + "isExpression": true + }, + "\"id\" desc": { + "isExpression": true + } + } + }, + "idx_extensions_id_nocase": { + "columns": { + "lower(\"id\")": { + "isExpression": true + } + } + }, + "idx_extensions_developer_order": { + "columns": { + "lower(\"id\")": { + "isExpression": true + } + } + }, + "idx_extensions_public_order": { + "columns": { + "lower(\"id\")": { + "isExpression": true + } + } + }, + "idx_extensions_type_public_order": { + "columns": { + "lower(\"id\")": { + "isExpression": true + } + } + } + } + } +} diff --git a/src/services/extensions/v2/db/migrations/meta/_journal.json b/src/services/extensions/v2/db/migrations/meta/_journal.json index 456cece5..a3a4fe24 100644 --- a/src/services/extensions/v2/db/migrations/meta/_journal.json +++ b/src/services/extensions/v2/db/migrations/meta/_journal.json @@ -113,6 +113,20 @@ "when": 1793793600000, "tag": "0029_align_manifest", "breakpoints": true + }, + { + "idx": 30, + "version": "6", + "when": 1793901703593, + "tag": "0030_directory_packages", + "breakpoints": true + }, + { + "idx": 31, + "version": "6", + "when": 1793931547450, + "tag": "0031_package_jobs", + "breakpoints": true } ] } diff --git a/src/services/extensions/v2/db/package-jobs.ts b/src/services/extensions/v2/db/package-jobs.ts new file mode 100644 index 00000000..0a943ac9 --- /dev/null +++ b/src/services/extensions/v2/db/package-jobs.ts @@ -0,0 +1,169 @@ +import type { PackageJob, PackageJobRepository } from "../packages/jobs"; +import { PackageError } from "../packages/errors"; + +interface JobRow { + id: string; + fingerprint: string; + owner: string; + manifest: string; + source_id: string | null; + upload_object_id: string | null; + upload_sha256: string | null; + status: PackageJob["status"]; + attempts: number; + artifact_id: string | null; + error_code: string | null; + error_message: string | null; + error_status: number | null; + created_at: string; + updated_at: string; +} +const job = (row: JobRow): PackageJob => ({ + id: row.id, + fingerprint: row.fingerprint, + owner: JSON.parse(row.owner), + manifest: JSON.parse(row.manifest), + sourceId: row.source_id, + uploadObjectId: row.upload_object_id, + uploadSha256: row.upload_sha256, + status: row.status, + attempts: row.attempts, + artifactId: row.artifact_id, + error: row.error_code + ? { + code: row.error_code, + message: row.error_message!, + status: row.error_status! + } + : null, + createdAt: row.created_at, + updatedAt: row.updated_at +}); + +export class PackageJobsDatabase implements PackageJobRepository { + constructor(private readonly db: D1Database) {} + + async get(id: string) { + const row = await this.db + .prepare("SELECT * FROM package_jobs WHERE id=?") + .bind(id) + .first(); + return row ? job(row) : null; + } + async find(fingerprint: string) { + const row = await this.db + .prepare( + "SELECT * FROM package_jobs WHERE fingerprint=? AND status!='failed'" + ) + .bind(fingerprint) + .first(); + return row ? job(row) : null; + } + async create( + input: Pick< + PackageJob, + | "fingerprint" + | "owner" + | "manifest" + | "sourceId" + | "uploadObjectId" + | "uploadSha256" + > + ) { + const id = crypto.randomUUID(); + await this.db + .prepare( + `INSERT INTO package_jobs (id,fingerprint,owner_user_id,owner,manifest,source_id,upload_object_id,upload_sha256) + SELECT ?,?,?,?,?,?,?,? WHERE (SELECT count(*) FROM package_jobs WHERE owner_user_id=?) < 256 + ON CONFLICT DO NOTHING` + ) + .bind( + id, + input.fingerprint, + input.owner.owner_user_id, + JSON.stringify(input.owner), + JSON.stringify(input.manifest), + input.sourceId, + input.uploadObjectId, + input.uploadSha256, + input.owner.owner_user_id + ) + .run(); + const result = await this.find(input.fingerprint); + if (!result) + throw new PackageError( + "PACKAGE_ADMISSION", + "Package job allowance is exhausted; try again later", + 429 + ); + return result; + } + async claim(id: string, lease: string) { + const row = await this.db + .prepare( + `UPDATE package_jobs SET status='processing',attempts=attempts+1,lease_id=?,lease_until=datetime('now','+20 minutes'),updated_at=CURRENT_TIMESTAMP + WHERE id=? AND (status='queued' OR status='processing' AND lease_until(); + return row ? job(row) : null; + } + async complete(id: string, lease: string, artifactId: string) { + await this.db + .prepare( + "UPDATE package_jobs SET status='ready',artifact_id=?,error_code=NULL,error_message=NULL,error_status=NULL,lease_id=NULL,lease_until=NULL,updated_at=CURRENT_TIMESTAMP WHERE id=? AND lease_id=?" + ) + .bind(artifactId, id, lease) + .run(); + } + async fail( + id: string, + lease: string, + error: NonNullable, + retry: boolean + ) { + await this.db + .prepare( + "UPDATE package_jobs SET status=?,error_code=?,error_message=?,error_status=?,lease_id=NULL,lease_until=NULL,updated_at=CURRENT_TIMESTAMP WHERE id=? AND lease_id=?" + ) + .bind( + retry ? "queued" : "failed", + error.code, + error.message, + error.status, + id, + lease + ) + .run(); + } + async pending() { + const result = await this.db + .prepare( + `SELECT id FROM package_jobs WHERE + (status='queued' AND (enqueued_at IS NULL OR enqueued_at(); + return result.results.map((row) => row.id); + } + async markEnqueued(id: string) { + await this.db + .prepare( + "UPDATE package_jobs SET enqueued_at=CURRENT_TIMESTAMP WHERE id=?" + ) + .bind(id) + .run(); + } + async expire() { + await this.db + .prepare( + `UPDATE package_jobs SET status='failed',error_code='PACKAGE_EXPIRED',error_message='Package processing expired. Please try again.',error_status=503,lease_id=NULL,lease_until=NULL,updated_at=CURRENT_TIMESTAMP WHERE status IN ('queued','processing') AND created_at v !== undefined) + .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) + .map(([k, v]) => `${JSON.stringify(k)}:${canonicalJson(v)}`) + .join(",")}}`; + return JSON.stringify(value); +} + +export function contentManifest(id: string, content: ExtensionContent) { + return ManifestSchema.parse({ + id, + manifest_version: content.manifest_version, + type: content.type, + name: content.name, + version: content.version, + description: content.description, + author: content.author, + license: content.license, + homepage_url: content.homepage_url, + requires: content.requires + }); +} + +export interface ArtifactRow { + id: string; + source_id: string; + manifest: string; + sha256: string; + object_id: string; + download_url: string; + published_at: string | null; + created_at: string; + extension_id: string; + profile_generation: string; + version: string; + icon_source_id: string | null; + avif_object_id: string | null; + avif_sha256: string | null; + png_object_id: string | null; + png_sha256: string | null; + svg_object_id: string | null; + svg_sha256: string | null; +} + +export const ARTIFACT_SELECT = `SELECT a.*, s.extension_id, s.profile_generation, s.version, i.source_id AS icon_source_id, i.avif_object_id, i.avif_sha256, i.png_object_id, i.png_sha256, i.svg_object_id, i.svg_sha256 + FROM package_artifacts a JOIN package_sources s ON s.id=a.source_id LEFT JOIN package_icons i ON i.source_id=s.id`; + +// All submission, publisher, correction and approval writers use this check. +export async function validatePackageContent( + db: D1Database, + id: string, + developerId: string | undefined, + content: ExtensionContent, + callerId?: string +): Promise { + id = id.toLowerCase(); + if (!developerId) { + const ref = await db + .prepare( + `SELECT d.id FROM extensions e JOIN developers d ON d.id=e.developer_id WHERE lower(e.id)=lower(?) AND (? IS NULL OR d.owner_user_id=?)` + ) + .bind(id, callerId ?? null, callerId ?? null) + .first<{ id: string }>(); + if (!ref) return null; + developerId = ref.id; + } + const published = await db + .prepare( + `SELECT DISTINCT s.version FROM package_artifacts a JOIN package_sources s ON s.id=a.source_id WHERE s.extension_id=? AND a.published_at IS NOT NULL` + ) + .bind(id) + .all<{ version: string }>(); + const publishedVersions = new Set( + published.results.map((row) => row.version) + ); + const managed = publishedVersions.size > 0; + const hasArtifacts = content.releases.some((r) => r.artifact_id); + const selectedCount = content.releases.filter( + (r) => r.is_latest === true + ).length; + const fail = (message: string): DatabaseError => ({ + code: "CONFLICT", + message + }); + if (selectedCount > 1) return fail("Select only one latest release"); + if ( + !managed && + !hasArtifacts && + !selectedCount && + !content.releases.some((r) => r.sha256) + ) + return null; + const latest = latestRelease(content.releases); + if ( + !latest || + latest.tag.replace(/^v/, "") !== content.version || + latest.download_url !== content.download_url || + canonicalJson(latest.requires) !== canonicalJson(content.requires) + ) + return fail( + "Version, download and requirements must match the selected latest release" + ); + if ((managed || hasArtifacts) && !latest.artifact_id) + return fail( + "The latest release needs a directory-built package; upload its source ZIP first" + ); + const versions = new Set(); + for (const release of content.releases) { + const version = release.tag.replace(/^v/, ""); + if ((managed || hasArtifacts) && versions.has(version)) + return fail("Duplicate release version"); + versions.add(version); + if (!release.artifact_id) { + if (release.sha256) + return fail("Checksums require a directory-built artifact"); + if (publishedVersions.has(version)) + return fail( + "A directory-built release cannot be replaced with an external download" + ); + continue; + } + const artifact = await db + .prepare( + `${ARTIFACT_SELECT} JOIN developers d ON d.profile_generation=s.profile_generation WHERE a.id=? AND s.extension_id=? AND d.id=?` + ) + .bind(release.artifact_id, id, developerId) + .first(); + if ( + !artifact || + artifact.version !== version || + artifact.download_url !== release.download_url || + artifact.sha256 !== release.sha256 + ) + return fail("Release does not match its package artifact"); + const manifest = ManifestSchema.parse(JSON.parse(artifact.manifest)); + if (canonicalJson(manifest.requires) !== canonicalJson(release.requires)) + return fail("Release requirements do not match its package manifest"); + if ( + release === latest && + (content.download_url !== artifact.download_url || + canonicalJson(manifest) !== canonicalJson(contentManifest(id, content))) + ) + return fail( + "Overview metadata must match the latest package manifest; rebuild it before submitting" + ); + } + return null; +} + +export function packageIconUrl(content: ExtensionContent): string | null { + return latestRelease(content.releases)?.artifact_id + ? content.download_url.replace(/\/download$/, "/icon") + : null; +} diff --git a/src/services/extensions/v2/db/revisions.ts b/src/services/extensions/v2/db/revisions.ts index bf9af00c..ba1983c2 100644 --- a/src/services/extensions/v2/db/revisions.ts +++ b/src/services/extensions/v2/db/revisions.ts @@ -1,3 +1,5 @@ +import { packageIconUrl } from "./packages"; +import { validatePackageContent } from "./packages"; import { manifestMetadata } from "../schemas/manifest"; import { MAX_CONTENT_BYTES } from "../resource-limits"; import { and, asc, desc, eq, gt, lt, or, sql, SQL } from "drizzle-orm"; @@ -148,6 +150,14 @@ export class ExtensionRevisionsDatabase { let result; try { + const packageError = await validatePackageContent( + this.db.$client, + input.extensionId, + undefined, + input.content, + input.callerId + ); + if (packageError) return { data: null, error: packageError }; result = await this.db.run(sql` INSERT INTO ${extensionRevisions} (id, extension_id, developer_id, submitted_by, status, content, ownership_epoch) @@ -192,6 +202,14 @@ export class ExtensionRevisionsDatabase { }): Promise> { const id = crypto.randomUUID(); try { + const packageError = await validatePackageContent( + this.db.$client, + input.extensionId, + undefined, + input.content, + input.callerId + ); + if (packageError) return { data: null, error: packageError }; const results = await this.db.$client.batch<{ extensionId: string }>([ this.db.$client .prepare( @@ -326,6 +344,13 @@ export class ExtensionRevisionsDatabase { let result; try { + const packageError = await validatePackageContent( + this.db.$client, + input.extensionId, + input.developerId, + input.content + ); + if (packageError) return { data: null, error: packageError }; result = await this.db.run(sql` INSERT INTO ${extensionRevisions} (id, extension_id, developer_id, submitted_by, status, content, ownership_epoch) @@ -780,6 +805,20 @@ export class ExtensionRevisionsDatabase { } }; } + try { + const packageError = await validatePackageContent( + this.db.$client, + revision.extension_id, + revision.developer_id, + parsed.data + ); + if (packageError) return { ok: false, error: packageError }; + } catch (error) { + return { + ok: false, + error: databaseError("preparePackageApproval", error).error! + }; + } return { ok: true, revision, content: parsed.data }; } @@ -823,7 +862,7 @@ export class ExtensionRevisionsDatabase { JSON.stringify(content.releases), content.homepage_url ?? "", JSON.stringify(content.license), - content.icon_url ?? null, + packageIconUrl(content), content.readme, JSON.stringify(content.source), content.version, diff --git a/src/services/extensions/v2/db/schema.ts b/src/services/extensions/v2/db/schema.ts index 8a1464e9..40f29b82 100644 --- a/src/services/extensions/v2/db/schema.ts +++ b/src/services/extensions/v2/db/schema.ts @@ -512,3 +512,133 @@ export const m2mPublisherClients = sqliteTable( ) ] ); + +// Object reservations survive interrupted uploads so unreferenced objects can +// be reclaimed. Published artifacts retain their original objects indefinitely. +export const packageObjects = sqliteTable( + "package_objects", + { + deleting: integer("deleting").notNull().default(0), + id: text("id").primaryKey(), + createdBy: text("created_by").notNull(), + bytes: integer("bytes").notNull(), + createdAt: text("created_at") + .notNull() + .default(sql`CURRENT_TIMESTAMP`) + }, + (table) => [index("idx_package_objects_creator").on(table.createdBy)] +); + +// No extension FK: a developer can build a package before creating its listing. +export const packageSources = sqliteTable( + "package_sources", + { + id: text("id").primaryKey(), + profileGeneration: text("profile_generation").notNull(), + extensionId: text("extension_id").notNull(), + version: text("version").notNull(), + sha256: text("sha256").notNull(), + objectId: text("object_id") + .notNull() + .references(() => packageObjects.id), + createdAt: text("created_at") + .notNull() + .default(sql`CURRENT_TIMESTAMP`) + }, + (table) => [ + uniqueIndex("idx_package_source_version").on( + table.profileGeneration, + table.extensionId, + table.version + ), + index("idx_package_sources_extension").on(table.extensionId), + uniqueIndex("idx_package_sources_object").on(table.objectId) + ] +); + +export const packageArtifacts = sqliteTable( + "package_artifacts", + { + id: text("id").primaryKey(), + sourceId: text("source_id") + .notNull() + .references(() => packageSources.id), + manifest: text("manifest").notNull(), + manifestHash: text("manifest_hash").notNull(), + sha256: text("sha256").notNull(), + objectId: text("object_id") + .notNull() + .references(() => packageObjects.id), + downloadUrl: text("download_url").notNull(), + publishedAt: text("published_at"), + createdAt: text("created_at") + .notNull() + .default(sql`CURRENT_TIMESTAMP`) + }, + (table) => [ + uniqueIndex("idx_package_artifact_manifest").on( + table.sourceId, + table.manifestHash + ), + uniqueIndex("idx_package_artifacts_object").on(table.objectId) + ] +); + +// Prepared display images are shared by packages using the same immutable source. +export const packageIcons = sqliteTable("package_icons", { + sourceId: text("source_id") + .primaryKey() + .references(() => packageSources.id), + avifObjectId: text("avif_object_id").references(() => packageObjects.id), + avifSha256: text("avif_sha256"), + pngObjectId: text("png_object_id").references(() => packageObjects.id), + pngSha256: text("png_sha256"), + svgObjectId: text("svg_object_id").references(() => packageObjects.id), + svgSha256: text("svg_sha256") +}); + +export const packageJobs = sqliteTable( + "package_jobs", + { + id: text("id").primaryKey(), + fingerprint: text("fingerprint").notNull(), + ownerUserId: text("owner_user_id").notNull(), + owner: text("owner").notNull(), + manifest: text("manifest").notNull(), + sourceId: text("source_id"), + uploadObjectId: text("upload_object_id").references( + () => packageObjects.id + ), + uploadSha256: text("upload_sha256"), + status: text("status", { + enum: ["queued", "processing", "ready", "failed"] + }) + .notNull() + .default("queued"), + attempts: integer("attempts").notNull().default(0), + leaseId: text("lease_id"), + leaseUntil: text("lease_until"), + artifactId: text("artifact_id").references(() => packageArtifacts.id), + errorCode: text("error_code"), + errorMessage: text("error_message"), + errorStatus: integer("error_status"), + enqueuedAt: text("enqueued_at"), + createdAt: text("created_at") + .notNull() + .default(sql`CURRENT_TIMESTAMP`), + updatedAt: text("updated_at") + .notNull() + .default(sql`CURRENT_TIMESTAMP`) + }, + (table) => [ + uniqueIndex("idx_package_jobs_fingerprint") + .on(table.fingerprint) + .where(sql`${table.status} != 'failed'`), + index("idx_package_jobs_owner").on(table.ownerUserId), + index("idx_package_jobs_dispatch").on(table.status, table.enqueuedAt), + check( + "package_jobs_status_check", + sql`${table.status} IN ('queued','processing','ready','failed')` + ) + ] +); diff --git a/src/services/extensions/v2/index.ts b/src/services/extensions/v2/index.ts index 948fcc8f..ce7a76b0 100644 --- a/src/services/extensions/v2/index.ts +++ b/src/services/extensions/v2/index.ts @@ -1,3 +1,4 @@ +import { registerPackageRoutes } from "./routes/packages"; import { boundContentRequest, paceContentIp, @@ -90,6 +91,7 @@ registerOwnershipRoutes(extensionsV2); registerModerationRoutes(extensionsV2); registerM2mRoutes(extensionsV2); registerRevisionRoutes(extensionsV2); +registerPackageRoutes(extensionsV2); registerDeveloperProfileRoutes(extensionsV2); // The document is deterministic once every route is registered, so build diff --git a/src/services/extensions/v2/package-builder.ts b/src/services/extensions/v2/package-builder.ts new file mode 100644 index 00000000..2b8d18c6 --- /dev/null +++ b/src/services/extensions/v2/package-builder.ts @@ -0,0 +1,206 @@ +import { + Uint8ArrayReader, + Uint8ArrayWriter, + ZipReader, + ZipWriter +} from "@zip.js/zip.js"; +import { ManifestSchema } from "./schemas/manifest"; + +// Bounds for uploaded ZIPs and their decompressed contents. +export const PACKAGE_LIMITS = { + sourceBytes: 5 * 1024 * 1024, + expandedBytes: 16 * 1024 * 1024, + entries: 500 +} as const; + +const zipOptions = { + useWebWorkers: false, + useCompressionStream: true, + strictness: "strict" as const, + checkCrc32: true, + checkOverlappingEntry: true +}; + +export async function sha256(bytes: Uint8Array) { + const digest = await crypto.subtle.digest("SHA-256", bytes); + return Array.from(new Uint8Array(digest), (byte) => + byte.toString(16).padStart(2, "0") + ).join(""); +} + +function validatePath(filename: string, directory: boolean) { + const path = directory ? filename.replace(/\/$/, "") : filename; + if ( + !path || + /[\\:<>"|?*]/.test(path) || + [...path].some( + (char) => char.charCodeAt(0) < 32 || char.charCodeAt(0) === 127 + ) || + path.split("/").some((part) => !part || part === "." || part === "..") || + path + .split("/") + .some( + (part) => + /[. ]$/.test(part) || + /^(con|prn|aux|nul|com[1-9]|lpt[1-9])(?:\.|$)/i.test(part) + ) + ) { + throw new Error(`Unsafe package path: ${filename}`); + } + return path; +} + +/** Validate source files and build a bounded archive with directory metadata. */ +export async function buildExtensionPackage( + source: Uint8Array, + manifestInput: unknown +) { + const manifest = ManifestSchema.parse(manifestInput); + if (source.byteLength > PACKAGE_LIMITS.sourceBytes) { + throw new Error("Source ZIP exceeds size limit"); + } + const reader = new ZipReader(new Uint8ArrayReader(source), zipOptions); + const files: { + path: string; + bytes: Uint8Array; + compression?: { + passThrough: true; + compressionMethod: number; + uncompressedSize: number; + crc32?: number; + }; + icon?: Uint8Array; + }[] = []; + const paths = new Map(); + let count = 0; + let expandedBytes = 0; + try { + for await (const entry of reader.getEntriesGenerator()) { + if (++count > PACKAGE_LIMITS.entries) { + throw new Error("Source ZIP exceeds entry limit"); + } + const path = validatePath(entry.filename, entry.directory); + // Avoid case-folding collisions across supported installation platforms. + const key = path.normalize("NFC").toLowerCase(); + if (key.startsWith("manifest.json/")) { + throw new Error("Manifest path is a directory"); + } + if (paths.has(key)) throw new Error(`Duplicate package path: ${path}`); + paths.set(key, entry.directory); + const fileType = (entry.externalFileAttributes >>> 16) & 0o170000; + if ( + entry.encrypted || + (fileType !== 0 && + fileType !== (entry.directory ? 0o040000 : 0o100000)) || + ![0, 8].includes(entry.compressionMethod) + ) { + throw new Error(`Unsupported package entry: ${path}`); + } + if ( + entry.uncompressedSize > + PACKAGE_LIMITS.expandedBytes - expandedBytes + ) { + throw new Error("Source ZIP exceeds expanded size limit"); + } + if (entry.directory) continue; + // Bound the actual decompressed bytes, not just the untrusted ZIP header. + const isIcon = /^icon\.(svg|png|webp|avif)$/.test(path); + const chunks: Uint8Array[] = []; + let length = 0; + await entry.getData( + new WritableStream({ + write(chunk) { + expandedBytes += chunk.byteLength; + length += chunk.byteLength; + if (expandedBytes > PACKAGE_LIMITS.expandedBytes) { + throw new Error("Source ZIP exceeds expanded size limit"); + } + if (isIcon) chunks.push(chunk.slice()); + } + }) + ); + if (length !== entry.uncompressedSize) { + throw new Error("Source ZIP entry has inconsistent size"); + } + const icon = isIcon ? new Uint8Array(length) : undefined; + let offset = 0; + for (const chunk of chunks) { + icon!.set(chunk, offset); + offset += chunk.byteLength; + } + // Validate the existing manifest's ZIP entry too, then replace its bytes. + if (key === "manifest.json") { + if (path !== "manifest.json") { + throw new Error("Manifest must be named manifest.json"); + } + } else { + // CRC and expanded-size validation above still decompresses every file. + // Copy its original compressed payload instead of compressing it again. + const bytes = await entry.getData(new Uint8ArrayWriter(), { + passThrough: true + }); + files.push({ + path, + bytes, + icon, + compression: { + passThrough: true, + compressionMethod: entry.compressionMethod, + uncompressedSize: entry.uncompressedSize, + crc32: entry.signature + } + }); + } + } + } finally { + await reader.close(); + } + for (const [path] of paths) { + const parts = path.split("/"); + parts.pop(); + while (parts.length) { + if (paths.get(parts.join("/")) === false) { + throw new Error(`File conflicts with package directory: ${path}`); + } + parts.pop(); + } + } + if (paths.get("manifest.json") === true) { + throw new Error("Manifest path is a directory"); + } + if (!files.length) throw new Error("Package contains no source files"); + const icons = files.filter((file) => + /^icon\.(svg|png|webp|avif)$/.test(file.path) + ); + if (icons.length > 1) throw new Error("Include only one package icon"); + + const manifestBytes = new Uint8Array( + new TextEncoder().encode(JSON.stringify(manifest, null, 2) + "\n") + ); + files.push({ path: "manifest.json", bytes: manifestBytes }); + files.sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0)); + const writer = new ZipWriter(new Uint8ArrayWriter(), { + ...zipOptions, + level: 6, + lastModDate: new Date("1980-01-01T00:00:00Z"), + extendedTimestamp: false + }); + for (const file of files) { + await writer.add( + file.path, + new Uint8ArrayReader(file.bytes), + file.compression + ); + } + const bytes = await writer.close(); + return { + bytes, + manifest, + sourceSha256: await sha256(source), + packageSha256: await sha256(bytes), + sourceFileCount: files.length - 1, + expandedBytes, + icon: icons[0]?.icon ?? null, + iconName: icons[0]?.path + }; +} diff --git a/src/services/extensions/v2/packages/b2-store.ts b/src/services/extensions/v2/packages/b2-store.ts new file mode 100644 index 00000000..503286ee --- /dev/null +++ b/src/services/extensions/v2/packages/b2-store.ts @@ -0,0 +1,130 @@ +import { AwsClient } from "aws4fetch"; +import { PACKAGE_LIMITS, sha256 } from "../package-builder"; +import { PackageError } from "./errors"; +import { boundedBytes, type PackageStore } from "./store"; + +// Includes generated manifest and ZIP headers as well as the source contents. +const MAX_STORED_BYTES = PACKAGE_LIMITS.expandedBytes + 1024 * 1024; + +export function b2PackageStore( + getEnv: (name: string) => string | undefined +): PackageStore { + const get = (suffix: string) => { + const value = getEnv(`EXTENSION_PACKAGES_S3_${suffix}`); + if (!value) + throw unavailable("Package storage configuration is incomplete"); + return value; + }; + const origin = new URL(get("ENDPOINT")); + const bucket = get("BUCKET"); + const prefix = getEnv("EXTENSION_PACKAGES_S3_PREFIX") ?? "local"; + if ( + origin.protocol !== "https:" || + origin.username || + origin.password || + origin.search || + origin.hash || + origin.pathname !== "/" || + !/^[a-zA-Z0-9-]+$/.test(bucket) || + !/^[a-zA-Z0-9_-]+$/.test(prefix) + ) + throw unavailable("Invalid package storage configuration"); + const aws = new AwsClient({ + accessKeyId: get("ACCESS_KEY_ID"), + secretAccessKey: get("SECRET_ACCESS_KEY"), + region: get("REGION"), + service: "s3", + retries: 0 + }); + function target(key: string) { + // Object IDs are allocated once in D1 before a write, never chosen by callers. + if ( + !/^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/.test( + key + ) + ) + throw unavailable("Invalid package object ID"); + return new URL(`/${bucket}/${prefix}/packages/${key}`, origin); + } + async function request(url: URL, init: RequestInit) { + try { + return await fetch( + await aws.sign(url, { + ...init, + redirect: "manual", + signal: AbortSignal.timeout(15000) + }) + ); + } catch { + throw unavailable("Package storage request failed"); + } + } + return { + async put(key, bytes, checksum) { + if ( + bytes.byteLength > MAX_STORED_BYTES || + (await sha256(bytes)) !== checksum + ) + throw unavailable("Package object failed checksum or size validation"); + const response = await request(target(key), { + method: "PUT", + body: bytes, + headers: { + "Content-Type": "application/zip", + "Cache-Control": "private, no-store", + "x-amz-content-sha256": checksum, + "x-amz-meta-sha256": checksum + } + }); + await response.body?.cancel(); + if (!response.ok) throw unavailable("Package upload failed"); + }, + async get(key, checksum) { + const response = await request(target(key), { method: "GET" }); + if (response.status === 404) { + await response.body?.cancel(); + return null; + } + if ( + response.status !== 200 || + (checksum && response.headers.get("x-amz-meta-sha256") !== checksum) + ) { + await response.body?.cancel(); + throw unavailable( + "Stored package is unavailable or has a different checksum" + ); + } + // Verify the actual bytes before returning a downloadable ZIP. The size + // bound also applies to responses without Content-Length (e.g. chunked). + let bytes: Uint8Array; + try { + bytes = await boundedBytes(response.body, MAX_STORED_BYTES); + } catch { + throw unavailable( + "Stored package could not be read within its size limit" + ); + } + if (checksum && (await sha256(bytes)) !== checksum) + throw unavailable("Stored package failed checksum verification"); + return new Blob([bytes]).stream(); + }, + async delete(key) { + const url = target(key); + const head = await request(url, { method: "HEAD" }); + await head.body?.cancel(); + if (head.status === 404) return; + const version = head.headers.get("x-amz-version-id"); + if (!head.ok || !version) + throw unavailable("Unable to identify package version for cleanup"); + // B2 DELETE without versionId only adds a marker and keeps billable bytes. + url.searchParams.set("versionId", version); + const response = await request(url, { method: "DELETE" }); + await response.body?.cancel(); + if (!response.ok && response.status !== 404) + throw unavailable("Package cleanup failed"); + } + }; +} +function unavailable(message: string) { + return new PackageError("PACKAGE_UNAVAILABLE", message, 503); +} diff --git a/src/services/extensions/v2/packages/build.ts b/src/services/extensions/v2/packages/build.ts new file mode 100644 index 00000000..7dbdf997 --- /dev/null +++ b/src/services/extensions/v2/packages/build.ts @@ -0,0 +1,474 @@ +import type { PublisherContext } from "../middleware"; +import { sanitizeSvg } from "@zeropress/svg-hush/workerd"; +import { + buildExtensionPackage, + PACKAGE_LIMITS, + sha256 +} from "../package-builder"; +import { Manifest } from "../schemas/manifest"; +import { ARTIFACT_SELECT, ArtifactRow, canonicalJson } from "../db/packages"; +import { PackageError } from "./errors"; +import { boundedBytes, PackageStore } from "./store"; + +export interface PackageOwner { + id: string; + profile_generation: string; + ownership_epoch: number; + owner_user_id: string; + publisher_client_id?: string; +} +interface SourceRow { + id: string; + sha256: string; + object_id: string; +} +export const MAX_PACKAGE_STORAGE_BYTES = 512 * 1024 * 1024; +export const MAX_PACKAGE_OBJECTS = 256; + +function ownerPredicate() { + return `EXISTS (SELECT 1 FROM developers d JOIN users u ON u.id=d.owner_user_id WHERE d.id=? AND d.profile_generation=? AND d.ownership_epoch=? AND d.owner_user_id=? AND u.deleted_at IS NULL AND NOT EXISTS (SELECT 1 FROM extensions e WHERE lower(e.id)=lower(?) AND e.developer_id!=d.id) AND (? IS NULL OR EXISTS (SELECT 1 FROM m2m_publisher_clients p WHERE p.client_id=? AND p.developer_id=d.id AND p.ownership_epoch=d.ownership_epoch AND p.revoked_at IS NULL)))`; +} +function ownerParams(owner: PackageOwner, id: string) { + return [ + owner.id, + owner.profile_generation, + owner.ownership_epoch, + owner.owner_user_id, + id, + owner.publisher_client_id ?? null, + owner.publisher_client_id ?? null + ]; +} + +export async function packageOwner( + db: D1Database, + userId: string, + extensionId: string, + publisher?: PublisherContext | null +) { + const owner = await db + .prepare( + `SELECT d.id,d.profile_generation,d.ownership_epoch,d.owner_user_id FROM developers d JOIN users u ON u.id=d.owner_user_id WHERE (? IS NULL AND d.owner_user_id=? OR d.id=? AND d.ownership_epoch=? AND EXISTS (SELECT 1 FROM m2m_publisher_clients p WHERE p.client_id=? AND p.developer_id=d.id AND p.ownership_epoch=d.ownership_epoch AND p.revoked_at IS NULL)) AND u.deleted_at IS NULL AND NOT EXISTS (SELECT 1 FROM extensions e WHERE lower(e.id)=lower(?) AND e.developer_id!=d.id)` + ) + .bind( + publisher?.clientId ?? null, + userId, + publisher?.developerId ?? null, + publisher?.ownershipEpoch ?? null, + publisher?.clientId ?? null, + extensionId + ) + .first(); + if (!owner) + throw new PackageError( + "FORBIDDEN", + "An active owning developer profile is required", + 403 + ); + return { ...owner, publisher_client_id: publisher?.clientId }; +} + +export async function reserveObject( + db: D1Database, + store: PackageStore, + owner: PackageOwner, + extensionId: string, + bytes: Uint8Array, + checksum: string, + suffix?: "avif" | "png" | "svg" +) { + const id = suffix + ? `icons/${crypto.randomUUID()}.${suffix}` + : crypto.randomUUID(); + const result = await db + .prepare( + `INSERT INTO package_objects (id,created_by,bytes) + SELECT ?,?,? WHERE ${ownerPredicate()} + AND (SELECT count(*) FROM package_objects WHERE created_by=?) < ? + AND (SELECT coalesce(sum(bytes),0) FROM package_objects WHERE created_by=?) + ? <= ?` + ) + .bind( + id, + owner.owner_user_id, + bytes.length, + ...ownerParams(owner, extensionId), + owner.owner_user_id, + MAX_PACKAGE_OBJECTS, + owner.owner_user_id, + bytes.length, + MAX_PACKAGE_STORAGE_BYTES + ) + .run(); + if (!result.meta.changes) + throw new PackageError( + "PACKAGE_ADMISSION", + "Ownership changed or package storage quota is exhausted", + 409 + ); + // A reservation is retained on failure/interruption for safe later cleanup. + await store.put(id, bytes, checksum); + return id; +} + +export async function cleanupPackageObjects( + db: D1Database, + store: PackageStore +) { + const result = await db + .prepare( + `UPDATE package_objects SET deleting=1 WHERE id IN ( + SELECT o.id FROM package_objects o WHERE (o.deleting=1 OR o.created_at < datetime('now','-1 day')) + AND NOT EXISTS (SELECT 1 FROM package_jobs j WHERE j.upload_object_id=o.id AND j.status IN ('queued','processing')) + AND NOT EXISTS (SELECT 1 FROM package_sources s WHERE s.object_id=o.id) + AND NOT EXISTS (SELECT 1 FROM package_artifacts a WHERE a.object_id=o.id) + AND NOT EXISTS (SELECT 1 FROM package_icons i WHERE i.avif_object_id=o.id OR i.png_object_id=o.id OR i.svg_object_id=o.id) LIMIT 100 + ) RETURNING id` + ) + .all<{ id: string }>(); + for (const { id } of result.results) { + await store.delete(id); + await db + .prepare( + "UPDATE package_jobs SET upload_object_id=NULL WHERE upload_object_id=? AND status IN ('ready','failed')" + ) + .bind(id) + .run(); + await db + .prepare("DELETE FROM package_objects WHERE id=? AND deleting=1") + .bind(id) + .run(); + } + return result.results.length; +} + +export async function buildStoredPackage(input: { + db: D1Database; + store: PackageStore; + owner: PackageOwner; + manifest: Manifest; + source?: Uint8Array; + sourceObjectId?: string; + sourceId?: string; + baseUrl: string; + images?: ImagesBinding; +}) { + const { db, store, owner, manifest } = input; + let source = await db + .prepare( + "SELECT id,sha256,object_id FROM package_sources WHERE profile_generation=? AND extension_id=? AND version=?" + ) + .bind(owner.profile_generation, manifest.id, manifest.version) + .first(); + if (input.sourceId && source?.id !== input.sourceId) + throw new PackageError( + "NOT_FOUND", + "Source not found for this extension version", + 404 + ); + let sourceBytes = input.source; + if (sourceBytes && source && (await sha256(sourceBytes)) !== source.sha256) + throw new PackageError( + "SOURCE_CHANGED", + "This version already has a different source ZIP; use a new extension version", + 409 + ); + const manifestJson = canonicalJson(manifest); + const manifestHash = await sha256( + new Uint8Array(new TextEncoder().encode(manifestJson)) + ); + if (source) { + const existing = await db + .prepare(`${ARTIFACT_SELECT} WHERE a.source_id=? AND a.manifest_hash=?`) + .bind(source.id, manifestHash) + .first(); + if (existing) return existing; + } + if (!sourceBytes) { + if (!source) + throw new PackageError("NOT_FOUND", "Source ZIP not found", 404); + sourceBytes = await boundedBytes( + await store.get(source.object_id), + PACKAGE_LIMITS.sourceBytes + ); + if ((await sha256(sourceBytes)) !== source.sha256) + throw new PackageError( + "SOURCE_UNAVAILABLE", + "Stored source failed checksum verification", + 503 + ); + } + let built; + try { + built = await buildExtensionPackage(sourceBytes, manifest); + } catch { + throw new PackageError( + "INVALID_PACKAGE", + "Source ZIP failed package validation", + 422 + ); + } + const prepared = + source && + (await db + .prepare("SELECT 1 FROM package_icons WHERE source_id=?") + .bind(source.id) + .first()); + const variants = prepared + ? [] + : await optimizePackageIcon( + built.icon, + input.images, + built.iconName === "icon.svg" + ); + if (!source) { + const objectId = + input.sourceObjectId ?? + (await reserveObject( + db, + store, + owner, + manifest.id, + sourceBytes, + built.sourceSha256 + )); + const sourceId = crypto.randomUUID(); + await db + .prepare( + `INSERT INTO package_sources (id,profile_generation,extension_id,version,sha256,object_id) + SELECT ?,?,?,?,?,? WHERE ${ownerPredicate()} AND EXISTS (SELECT 1 FROM package_objects WHERE id=? AND deleting=0) + ON CONFLICT DO NOTHING` + ) + .bind( + sourceId, + owner.profile_generation, + manifest.id, + manifest.version, + built.sourceSha256, + objectId, + ...ownerParams(owner, manifest.id), + objectId + ) + .run(); + source = await db + .prepare( + "SELECT id,sha256,object_id FROM package_sources WHERE profile_generation=? AND extension_id=? AND version=?" + ) + .bind(owner.profile_generation, manifest.id, manifest.version) + .first(); + if (!source) + throw new PackageError( + "FORBIDDEN", + "Ownership changed while building", + 403 + ); + if (source.object_id !== objectId) { + if (!input.sourceObjectId) { + await store.delete(objectId); + await db + .prepare("DELETE FROM package_objects WHERE id=?") + .bind(objectId) + .run(); + } + if (source.sha256 !== built.sourceSha256) + throw new PackageError( + "SOURCE_CHANGED", + "Another upload reserved this version with different source files", + 409 + ); + } + } + await storePackageIcon({ + db, + store, + owner, + manifest, + sourceId: source.id, + variants + }); + const objectId = await reserveObject( + db, + store, + owner, + manifest.id, + built.bytes, + built.packageSha256 + ); + const artifactId = crypto.randomUUID(); + const downloadUrl = `${input.baseUrl}/packages/${artifactId}/download`; + await db + .prepare( + `INSERT INTO package_artifacts (id,source_id,manifest,manifest_hash,sha256,object_id,download_url) + SELECT ?,?,?,?,?,?,? WHERE ${ownerPredicate()} AND EXISTS (SELECT 1 FROM package_objects WHERE id=? AND deleting=0) + ON CONFLICT DO NOTHING` + ) + .bind( + artifactId, + source.id, + manifestJson, + manifestHash, + built.packageSha256, + objectId, + downloadUrl, + ...ownerParams(owner, manifest.id), + objectId + ) + .run(); + const artifact = await db + .prepare(`${ARTIFACT_SELECT} WHERE a.source_id=? AND a.manifest_hash=?`) + .bind(source.id, manifestHash) + .first(); + if (!artifact) + throw new PackageError( + "FORBIDDEN", + "Ownership changed while building", + 403 + ); + if (artifact.object_id !== objectId) { + await store.delete(objectId); + await db + .prepare("DELETE FROM package_objects WHERE id=?") + .bind(objectId) + .run(); + } + return artifact; +} + +async function storePackageIcon(input: { + db: D1Database; + store: PackageStore; + owner: PackageOwner; + manifest: Manifest; + sourceId: string; + variants: IconVariant[]; +}) { + const { db, store, owner, manifest, sourceId, variants } = input; + if ( + await db + .prepare("SELECT 1 FROM package_icons WHERE source_id=?") + .bind(sourceId) + .first() + ) + return; + const objects: { + format: IconVariant["format"]; + id: string; + checksum: string; + }[] = []; + for (const { format, bytes } of variants) { + const checksum = await sha256(bytes); + objects.push({ + format, + id: await reserveObject( + db, + store, + owner, + manifest.id, + bytes, + checksum, + format + ), + checksum + }); + } + const avif = objects.find((object) => object.format === "avif"); + const png = objects.find((object) => object.format === "png"); + const svg = objects.find((object) => object.format === "svg"); + const result = await db + .prepare( + `INSERT INTO package_icons (source_id,avif_object_id,avif_sha256,png_object_id,png_sha256,svg_object_id,svg_sha256) + SELECT ?,?,?,?,?,?,? WHERE ${ownerPredicate()} + AND NOT EXISTS (SELECT 1 FROM package_objects WHERE id IN (?,?,?) AND deleting=1) + ON CONFLICT DO NOTHING` + ) + .bind( + sourceId, + avif?.id ?? null, + avif?.checksum ?? null, + png?.id ?? null, + png?.checksum ?? null, + svg?.id ?? null, + svg?.checksum ?? null, + ...ownerParams(owner, manifest.id), + avif?.id ?? null, + png?.id ?? null, + svg?.id ?? null + ) + .run(); + if (!result.meta.changes) { + for (const object of objects) { + await store.delete(object.id); + await db + .prepare("DELETE FROM package_objects WHERE id=?") + .bind(object.id) + .run(); + } + if ( + !(await db + .prepare("SELECT 1 FROM package_icons WHERE source_id=?") + .bind(sourceId) + .first()) + ) + throw new PackageError( + "FORBIDDEN", + "Ownership changed while processing icon", + 403 + ); + } +} + +type IconVariant = { + format: "avif" | "png" | "svg"; + bytes: Uint8Array; +}; +async function optimizePackageIcon( + icon: Uint8Array | null, + images?: ImagesBinding, + isSvg = false +): Promise { + if (!icon) return []; + if (isSvg) { + try { + return [{ format: "svg", bytes: new Uint8Array(sanitizeSvg(icon)) }]; + } catch { + throw new PackageError( + "INVALID_ICON", + "Package SVG icon could not be sanitized", + 422 + ); + } + } + if (!images) + throw new PackageError( + "PACKAGES_DISABLED", + "Image processing is not configured", + 503 + ); + const variants: IconVariant[] = []; + try { + const info = await images.info(new Blob([icon]).stream()); + // Check decoded format too: a misleading filename must not bypass sanitization. + if (info.format === "image/svg+xml") + return [{ format: "svg", bytes: new Uint8Array(sanitizeSvg(icon)) }]; + for (const format of ["avif", "png"] as const) { + const output = await images + .input(new Blob([icon]).stream()) + .transform({ + width: 256, + height: 256, + fit: "scale-down" + }) + .output({ format: `image/${format}`, anim: false }); + variants.push({ + format, + bytes: await boundedBytes(output.image(), 1024 * 1024) + }); + } + } catch { + throw new PackageError( + "INVALID_ICON", + "Package icon could not be processed", + 422 + ); + } + return variants; +} diff --git a/src/services/extensions/v2/packages/cloudflare-queue.ts b/src/services/extensions/v2/packages/cloudflare-queue.ts new file mode 100644 index 00000000..fad14fc3 --- /dev/null +++ b/src/services/extensions/v2/packages/cloudflare-queue.ts @@ -0,0 +1,108 @@ +import { createCloudflareBindings } from "../../../../lib/adapters/cloudflare"; +import { CloudflareQueueAdapter } from "../../../../lib/adapters/cloudflare/queue"; +import { PackageJobsDatabase } from "../db/package-jobs"; +import { buildStoredPackage, packageOwner } from "./build"; +import { PackageError } from "./errors"; +import { boundedBytes, configuredPackageStore } from "./store"; +import { processPackageJob, dispatchPackageJob, type PackageJob } from "./jobs"; +import { PACKAGE_LIMITS } from "../package-builder"; + +export async function prepareQueuedPackage( + env: CloudflareBindings, + job: PackageJob +): Promise { + const getEnv = (name: string) => + createCloudflareBindings(env).environment.get(name); + const store = configuredPackageStore(env, getEnv); + if (!store) + throw new PackageError( + "PACKAGES_DISABLED", + "Package storage is not configured", + 503 + ); + const previous = job.owner; + const current = await packageOwner( + env.DB_EXTENSIONS, + previous.owner_user_id, + job.manifest.id, + previous.publisher_client_id + ? { + clientId: previous.publisher_client_id, + developerId: previous.id, + ownershipEpoch: previous.ownership_epoch, + canPublish: false + } + : null + ); + if ( + current.id !== previous.id || + current.profile_generation !== previous.profile_generation || + current.ownership_epoch !== previous.ownership_epoch + ) + throw new PackageError( + "FORBIDDEN", + "Package ownership changed while queued", + 403 + ); + const source = job.uploadObjectId + ? await boundedBytes( + await store.get(job.uploadObjectId, job.uploadSha256 ?? undefined), + PACKAGE_LIMITS.sourceBytes + ) + : undefined; + const artifact = await buildStoredPackage({ + db: env.DB_EXTENSIONS, + store, + owner: current, + manifest: job.manifest, + source, + sourceObjectId: job.uploadObjectId ?? undefined, + sourceId: job.sourceId ?? undefined, + baseUrl: getEnv("EXTENSION_PACKAGES_BASE_URL")!.replace(/\/$/, ""), + images: env.PACKAGE_IMAGES + }); + return artifact.id; +} + +export async function consumePackageJobs( + batch: MessageBatch<{ jobId: string }>, + env: CloudflareBindings +) { + const jobs = new PackageJobsDatabase(env.DB_EXTENSIONS); + for (const message of batch.messages) { + if ( + !message.body || + typeof message.body.jobId !== "string" || + !/^[a-f0-9-]{36}$/.test(message.body.jobId) + ) { + message.ack(); + continue; + } + try { + const result = await processPackageJob(jobs, message.body.jobId, (job) => + prepareQueuedPackage(env, job) + ); + if (result === "busy") message.retry({ delaySeconds: 1200 }); + else if (result === "retry") message.retry({ delaySeconds: 60 }); + else message.ack(); + } catch { + message.retry({ delaySeconds: 60 }); + } + } +} + +export async function recoverPackageJobs(env: CloudflareBindings) { + const mode = createCloudflareBindings(env).environment.get( + "EXTENSION_PACKAGES_MODE" + ); + if ( + !env.EXTENSION_PACKAGE_QUEUE || + !["local", "enabled"].includes(mode ?? "") + ) + return; + const jobs = new PackageJobsDatabase(env.DB_EXTENSIONS); + await jobs.expire(); + const queue = new CloudflareQueueAdapter(env.EXTENSION_PACKAGE_QUEUE); + for (const id of await jobs.pending()) + await dispatchPackageJob(jobs, queue, id); +} diff --git a/src/services/extensions/v2/packages/errors.ts b/src/services/extensions/v2/packages/errors.ts new file mode 100644 index 00000000..333a574d --- /dev/null +++ b/src/services/extensions/v2/packages/errors.ts @@ -0,0 +1,9 @@ +export class PackageError extends Error { + constructor( + public code: string, + message: string, + public status: 403 | 404 | 409 | 413 | 422 | 429 | 503 + ) { + super(message); + } +} diff --git a/src/services/extensions/v2/packages/jobs.ts b/src/services/extensions/v2/packages/jobs.ts new file mode 100644 index 00000000..d80c5153 --- /dev/null +++ b/src/services/extensions/v2/packages/jobs.ts @@ -0,0 +1,81 @@ +import type { IMessageQueue } from "../../../../lib/interfaces"; +import type { Manifest } from "../schemas/manifest"; +import type { PackageOwner } from "./build"; +import { PackageError } from "./errors"; + +export interface PackageJob { + id: string; + fingerprint: string; + owner: PackageOwner; + manifest: Manifest; + sourceId: string | null; + uploadObjectId: string | null; + uploadSha256: string | null; + status: "queued" | "processing" | "ready" | "failed"; + attempts: number; + artifactId: string | null; + error: { code: string; message: string; status: number } | null; + createdAt: string; + updatedAt: string; +} + +export interface PackageJobRepository { + get(id: string): Promise; + claim(id: string, lease: string): Promise; + complete(id: string, lease: string, artifactId: string): Promise; + fail( + id: string, + lease: string, + error: NonNullable, + retry: boolean + ): Promise; + pending(): Promise; + markEnqueued(id: string): Promise; +} + +export async function dispatchPackageJob( + jobs: PackageJobRepository, + queue: IMessageQueue, + id: string +) { + await queue.send({ jobId: id }); + await jobs.markEnqueued(id); +} + +// Delivery-specific acknowledgement/retry APIs stay in the platform adapter. +export async function processPackageJob( + jobs: PackageJobRepository, + id: string, + prepare: (job: PackageJob) => Promise, + lease = crypto.randomUUID() +): Promise<"ack" | "retry" | "busy"> { + const existing = await jobs.get(id); + if (!existing || existing.status === "ready" || existing.status === "failed") + return "ack"; + const job = await jobs.claim(id, lease); + if (!job) return "busy"; + try { + if (job.attempts > 3) + throw new PackageError( + "PACKAGE_UNAVAILABLE", + "Package processing was interrupted repeatedly. Please try again.", + 503 + ); + const artifactId = await prepare(job); + await jobs.complete(id, lease, artifactId); + return "ack"; + } catch (cause) { + const error = + cause instanceof PackageError + ? { code: cause.code, message: cause.message, status: cause.status } + : { + code: "PACKAGE_UNAVAILABLE", + message: + "Package processing is temporarily unavailable. Please try again.", + status: 503 + }; + const retry = error.status === 503 && job.attempts < 3; + await jobs.fail(id, lease, error, retry); + return retry ? "retry" : "ack"; + } +} diff --git a/src/services/extensions/v2/packages/public-assets.ts b/src/services/extensions/v2/packages/public-assets.ts new file mode 100644 index 00000000..718d7cab --- /dev/null +++ b/src/services/extensions/v2/packages/public-assets.ts @@ -0,0 +1,201 @@ +import { revalidateExtensions } from "../revalidate"; +import type { ExtensionContent } from "../schemas/extensions"; +import { sha256 } from "../package-builder"; +import { AwsClient } from "aws4fetch"; +import type { Context } from "hono"; +import { getPlatform } from "../../../../lib/middleware"; +import { ARTIFACT_SELECT, type ArtifactRow } from "../db/packages"; +import { latestRelease } from "../../../../lib/releases"; +import { PackageError } from "./errors"; +import { boundedBytes, packageStore } from "./store"; + +// Only called for committed, published content. Uploads and pending previews +// always use the private image bucket, even when B2 is configured. +export async function publishExtensionIcon( + c: Context<{ Bindings: CloudflareBindings }>, + extensionId: string +) { + const platform = getPlatform(c); + const endpoint = platform.getEnv("EXTENSION_ASSETS_S3_ENDPOINT"); + if (!endpoint) return; + const get = (name: string) => { + const value = platform.getEnv(`EXTENSION_ASSETS_${name}`); + if (!value) + throw new PackageError( + "ASSETS_UNAVAILABLE", + "Asset storage configuration is incomplete", + 503 + ); + return value; + }; + const row = await c.env.DB_EXTENSIONS.prepare( + "SELECT releases,published_revision_id FROM extensions WHERE id=? AND published_at IS NOT NULL AND delisted_at IS NULL" + ) + .bind(extensionId) + .first<{ releases: string; published_revision_id: string }>(); + if (!row) return; + const latest = latestRelease( + JSON.parse(row.releases) as ExtensionContent["releases"] + ); + if (!latest?.artifact_id) return; + const artifact = await c.env.DB_EXTENSIONS.prepare( + `${ARTIFACT_SELECT} WHERE a.id=? AND a.published_at IS NOT NULL AND s.extension_id=?` + ) + .bind(latest.artifact_id, extensionId) + .first(); + if ( + !artifact?.icon_source_id || + (!artifact.svg_object_id && !artifact.png_object_id) + ) + return; + const store = packageStore(c); + if (!store) return; + const origin = new URL(endpoint); + const publicBase = new URL(get("PUBLIC_URL")); + if ( + origin.protocol !== "https:" || + publicBase.protocol !== "https:" || + origin.username || + origin.password || + publicBase.username || + publicBase.password || + origin.search || + publicBase.search || + origin.hash || + publicBase.hash || + origin.pathname !== "/" + ) + throw new PackageError( + "ASSETS_UNAVAILABLE", + "Asset storage URLs must use HTTPS without credentials or query parameters", + 503 + ); + const bucket = get("S3_BUCKET"); + if (!/^[a-zA-Z0-9-]+$/.test(bucket)) + throw new PackageError( + "ASSETS_UNAVAILABLE", + "Invalid asset bucket name", + 503 + ); + const prefix = platform.getEnv("EXTENSION_ASSETS_S3_PREFIX") ?? "local"; + if (!/^[a-zA-Z0-9_-]+$/.test(prefix)) + throw new PackageError("ASSETS_UNAVAILABLE", "Invalid asset prefix", 503); + const aws = new AwsClient({ + accessKeyId: get("S3_ACCESS_KEY_ID"), + secretAccessKey: get("S3_SECRET_ACCESS_KEY"), + region: get("S3_REGION"), + service: "s3", + retries: 0 + }); + const identity = + artifact.svg_sha256 ?? + (await sha256( + new Uint8Array( + new TextEncoder().encode( + `${artifact.avif_sha256}:${artifact.png_sha256}` + ) + ) + )); + let iconUrl: string | undefined; + for (const format of artifact.svg_object_id + ? (["svg"] as const) + : (["avif", "png"] as const)) { + const key = artifact[`${format}_object_id`]; + const checksum = artifact[`${format}_sha256`]; + if (!key || !checksum) + throw new PackageError( + "ASSETS_UNAVAILABLE", + "Prepared asset is unavailable", + 503 + ); + const objectKey = `${prefix}/icons/${identity}.${format}`; + const url = new URL(`${publicBase.href.replace(/\/$/, "")}/${objectKey}`) + .href; + const contentType = format === "svg" ? "image/svg+xml" : `image/${format}`; + const cacheKey = `extension-asset:${url}`; + if (!(await c.env.CACHE_KV.get(cacheKey))) { + const target = new URL(`/${bucket}/${objectKey}`, origin); + const head = await fetch( + await aws.sign(target, { + method: "HEAD", + redirect: "manual", + signal: AbortSignal.timeout(5000) + }) + ); + if ( + head.status !== 404 && + (!head.ok || head.headers.get("x-amz-meta-sha256") !== checksum) + ) { + await head.body?.cancel(); + throw new PackageError( + "ASSETS_UNAVAILABLE", + "Unable to verify public asset storage", + 503 + ); + } + if (head.status === 404) { + const bytes = await boundedBytes( + await store.get(key, checksum), + 4 * 1024 * 1024 + ); + if (!bytes.length) + throw new PackageError( + "ASSETS_UNAVAILABLE", + "Prepared asset is unavailable", + 503 + ); + const response = await fetch( + await aws.sign(target, { + method: "PUT", + body: bytes, + redirect: "manual", + signal: AbortSignal.timeout(5000), + headers: { + "Content-Type": contentType, + "Cache-Control": "public, max-age=31536000, immutable", + "x-amz-content-sha256": checksum, + "x-amz-meta-sha256": checksum + } + }) + ); + await response.body?.cancel(); + if (!response.ok) + throw new PackageError( + "ASSETS_UNAVAILABLE", + "Public asset upload failed", + 503 + ); + } + await head.body?.cancel(); + await c.env.CACHE_KV.put(cacheKey, "ready", { expirationTtl: 86400 }); + } + // PNG is the fallback URL; the directory offers its AVIF sibling via . + if (format !== "avif") iconUrl = url; + } + if (iconUrl) { + const updated = await c.env.DB_EXTENSIONS.prepare( + "UPDATE extensions SET icon_url=? WHERE id=? AND published_revision_id=? AND delisted_at IS NULL AND icon_url IS NOT ?" + ) + .bind(iconUrl, extensionId, row.published_revision_id, iconUrl) + .run(); + return Boolean(updated.meta.changes); + } +} + +export function scheduleIconPublication( + c: Context<{ Bindings: CloudflareBindings }>, + extensionId: string +) { + if (!getPlatform(c).getEnv("EXTENSION_ASSETS_S3_ENDPOINT")) return; + c.executionCtx.waitUntil( + publishExtensionIcon(c, extensionId) + .then((changed) => { + if (changed) revalidateExtensions(c); + }) + .catch(() => { + console.error( + "[packages] Icon publication failed; existing image delivery remains available" + ); + }) + ); +} diff --git a/src/services/extensions/v2/packages/store.ts b/src/services/extensions/v2/packages/store.ts new file mode 100644 index 00000000..4299f65e --- /dev/null +++ b/src/services/extensions/v2/packages/store.ts @@ -0,0 +1,162 @@ +import { PackageError } from "./errors"; +import type { Context } from "hono"; +import { b2PackageStore } from "./b2-store"; +import { getPlatform } from "../../../../lib/middleware"; + +export interface PackageStore { + put( + key: string, + bytes: Uint8Array, + checksum: string + ): Promise; + get( + key: string, + checksum?: string + ): Promise | null>; + delete(key: string): Promise; +} + +export function packageStore( + c: Context<{ Bindings: CloudflareBindings }> +): PackageStore | null { + return configuredPackageStore(c.env, (name) => getPlatform(c).getEnv(name)); +} + +export function configuredPackageStore( + env: CloudflareBindings, + getEnv: (name: string) => string | undefined +): PackageStore | null { + const mode = getEnv("EXTENSION_PACKAGES_MODE"); + const production = getEnv("ENVIRONMENT") === "production"; + if (mode !== "enabled" && !(mode === "local" && !production)) return null; + if (production) { + const base = getEnv("EXTENSION_PACKAGES_BASE_URL"); + let url: URL; + try { + url = new URL(base ?? ""); + } catch { + throw new PackageError( + "PACKAGES_DISABLED", + "Package download URL is not configured", + 503 + ); + } + if ( + url.protocol !== "https:" || + url.username || + url.password || + url.search || + url.hash || + url.pathname !== "/extensions/v2" || + !getEnv("EXTENSION_PACKAGES_S3_ENDPOINT") || + !getEnv("EXTENSION_PACKAGES_S3_PREFIX") || + getEnv("EXTENSION_PACKAGES_S3_PREFIX") === "local" || + !env.EXTENSION_IMAGE_BUCKET || + !env.PACKAGE_IMAGES + ) { + throw new PackageError( + "PACKAGES_DISABLED", + "Production package storage, images or download URL is not configured", + 503 + ); + } + } + const remote = getEnv("EXTENSION_PACKAGES_S3_ENDPOINT") + ? b2PackageStore((name) => getEnv(name)) + : null; + const packages = env.EXTENSION_PACKAGE_BUCKET; + const images = env.EXTENSION_IMAGE_BUCKET; + const bucketFor = (key: string) => { + const bucket = key.startsWith("icons/") ? images : packages; + if (!bucket) + throw new PackageError( + "PACKAGES_DISABLED", + "Package or image storage is not configured", + 503 + ); + return bucket; + }; + if (!remote && !packages) return null; + return { + async put(key, bytes, checksum) { + if (remote && !key.startsWith("icons/")) + return remote.put(key, bytes, checksum); + const result = await bucketFor(key).put(key, bytes, { + onlyIf: { etagDoesNotMatch: "*" }, + sha256: checksum, + httpMetadata: { + contentType: key.endsWith(".avif") + ? "image/avif" + : key.endsWith(".png") + ? "image/png" + : key.endsWith(".svg") + ? "image/svg+xml" + : "application/zip", + cacheControl: "private, no-store" + } + }); + if (!result) throw new Error("Package object already exists"); + }, + async get(key, checksum) { + if (remote && !key.startsWith("icons/")) return remote.get(key, checksum); + const object = await bucketFor(key).get(key); + if (!object) return null; + if (checksum) { + const actual = object.checksums.sha256; + const hex = actual + ? Array.from(new Uint8Array(actual), (byte) => + byte.toString(16).padStart(2, "0") + ).join("") + : null; + if (hex !== checksum) { + await object.body.cancel(); + throw new PackageError( + "PACKAGE_UNAVAILABLE", + "Stored object failed checksum verification", + 503 + ); + } + } + return object.body; + }, + async delete(key) { + if (remote && !key.startsWith("icons/")) return remote.delete(key); + await bucketFor(key).delete(key); + } + }; +} + +export async function boundedBytes( + body: ReadableStream | null, + limit: number +) { + if (!body) return new Uint8Array(0); + const reader = body.getReader(); + const chunks: Uint8Array[] = []; + let size = 0; + try { + for (;;) { + const { value, done } = await reader.read(); + if (done) break; + size += value.byteLength; + if (size > limit) { + await reader.cancel(); + throw new PackageError( + "BODY_TOO_LARGE", + "Upload exceeds size limit", + 413 + ); + } + chunks.push(value); + } + } finally { + reader.releaseLock(); + } + const bytes = new Uint8Array(size); + let offset = 0; + for (const chunk of chunks) { + bytes.set(chunk, offset); + offset += chunk.byteLength; + } + return bytes; +} diff --git a/src/services/extensions/v2/routes/moderation.ts b/src/services/extensions/v2/routes/moderation.ts index 55f71f2b..85f9fac1 100644 --- a/src/services/extensions/v2/routes/moderation.ts +++ b/src/services/extensions/v2/routes/moderation.ts @@ -1,3 +1,4 @@ +import { scheduleIconPublication } from "../packages/public-assets"; import { paceContentAccount, requireApproveAccess, @@ -118,6 +119,7 @@ export function registerModerationRoutes(app: ExtensionsV2App): void { const status = statusFromWriteErrorCode(error?.code); return c.json(errorBody(error, "Unable to approve revision"), status); } + scheduleIconPublication(c, id); const notified = await notifyAuthor(c, extDb, query, { kind: "revision-approved", extensionId: id, @@ -399,6 +401,7 @@ export function registerModerationRoutes(app: ExtensionsV2App): void { const status = statusFromContentWriteError(error?.code); return c.json(errorBody(error, "Unable to correct extension"), status); } + scheduleIconPublication(c, id); revalidateExtensions(c); return c.json( { diff --git a/src/services/extensions/v2/routes/packages.ts b/src/services/extensions/v2/routes/packages.ts new file mode 100644 index 00000000..7e6e093c --- /dev/null +++ b/src/services/extensions/v2/routes/packages.ts @@ -0,0 +1,525 @@ +import { PackageJobsDatabase } from "../db/package-jobs"; +import { dispatchPackageJob, type PackageJob } from "../packages/jobs"; +import { canonicalJson } from "../db/packages"; +import { sha256 } from "../package-builder"; +import { scheduleIconPublication } from "../packages/public-assets"; +import { createRoute, z } from "@hono/zod-openapi"; +import type { Context, MiddlewareHandler } from "hono"; +import { getAuth } from "../../../../lib/auth"; +import { getPlatform } from "../../../../lib/middleware"; +import { ARTIFACT_SELECT, ArtifactRow } from "../db/packages"; +import { + requireContentWrite, + getPublisher, + requireModerator, + optionalAuth, + getOptionalAuth, + paceContentAccount, + paceContentIp +} from "../middleware"; +import { + reserveObject, + packageOwner, + cleanupPackageObjects +} from "../packages/build"; +import { PackageError } from "../packages/errors"; +import { boundedBytes, packageStore } from "../packages/store"; +import { ManifestSchema } from "../schemas/manifest"; +import { + PackageArtifactSchema, + PackageJobSchema, + PackageJobParamSchema, + PackageBuildSchema, + PackageParamSchema +} from "../schemas/packages"; +import { errorResponse, authErrorResponses } from "../schemas/common"; +import { ExtensionsV2App } from "./app"; + +function failure(c: Context, error: unknown) { + if (error instanceof PackageError) + return c.json( + { error: { code: error.code, message: error.message } }, + error.status + ); + console.error("[packages] Package operation failed"); + return c.json( + { + error: { + code: "PACKAGE_UNAVAILABLE", + message: "Package storage is temporarily unavailable; retry the request" + } + }, + 503 + ); +} +function artifactResponse(row: ArtifactRow) { + return { + result: { + id: row.id, + source_id: row.source_id, + manifest: ManifestSchema.loose().parse(JSON.parse(row.manifest)), + sha256: row.sha256, + download_url: row.download_url, + published_at: row.published_at, + created_at: row.created_at + } + }; +} +async function jobResponse(db: D1Database, job: PackageJob) { + const artifact = job.artifactId + ? await db + .prepare(`${ARTIFACT_SELECT} WHERE a.id=?`) + .bind(job.artifactId) + .first() + : null; + return { + result: { + id: job.id, + extension_id: job.manifest.id, + status: job.status, + artifact: artifact ? artifactResponse(artifact).result : null, + error: job.status === "failed" ? job.error : null, + created_at: job.createdAt, + updated_at: job.updatedAt + } + }; +} + +const packageErrors = { + ...authErrorResponses(), + 404: errorResponse("Package not found"), + 409: errorResponse( + "Source version conflict, ownership changed, or storage quota exhausted" + ), + 413: errorResponse("Upload exceeds size limit"), + 422: errorResponse("Invalid manifest or source ZIP"), + 429: errorResponse("Write allowance exhausted"), + 503: errorResponse("Package storage is not configured or is unavailable") +}; + +export function registerPackageRoutes(app: ExtensionsV2App) { + const availability: MiddlewareHandler<{ + Bindings: CloudflareBindings; + }> = async (c, next) => { + c.header("Cache-Control", "private, no-store"); + c.header("Vary", "Authorization"); + try { + if (!packageStore(c)) + throw new PackageError( + "PACKAGES_DISABLED", + "Package storage is not configured", + 503 + ); + } catch (error) { + return failure(c, error); + } + await next(); + }; + app.use("/packages", availability); + app.use("/packages/*", availability); + app.use("/package-jobs/*", availability); + app.on("POST", "/packages", paceContentIp()); + const boundUpload: MiddlewareHandler = async (c, next) => { + try { + const body = await boundedBytes(c.req.raw.body, 6 * 1024 * 1024); + c.req.raw = new Request(c.req.raw, { body }); + } catch (error) { + return failure(c, error); + } + await next(); + }; + + app.openapi( + createRoute({ + method: "post", + path: "/packages", + tags: ["Packages"], + summary: + "Queue package preparation from a distribution ZIP or stored source", + security: [{ Bearer: [] }], + middleware: [ + requireContentWrite(), + paceContentAccount(), + boundUpload + ] as const, + request: { + body: { + required: true, + content: { "multipart/form-data": { schema: PackageBuildSchema } } + } + }, + responses: { + 202: { + description: + "Private processing job; identical requests reuse queued or completed work", + content: { + "application/json": { + schema: z.object({ result: PackageJobSchema }) + } + } + }, + ...packageErrors + } + }), + async (c) => { + try { + const store = packageStore(c)!; + const form = c.req.valid("form"); + let json: unknown; + try { + json = JSON.parse(form.manifest); + } catch { + throw new PackageError( + "INVALID_MANIFEST", + "Manifest must be valid JSON", + 422 + ); + } + const parsed = ManifestSchema.safeParse(json); + if (!parsed.success) + throw new PackageError( + "INVALID_MANIFEST", + "Manifest failed validation", + 422 + ); + const manifest = parsed.data; + if (form.source && form.source.size > 5 * 1024 * 1024) + throw new PackageError( + "BODY_TOO_LARGE", + "Source ZIP exceeds 5 MiB", + 413 + ); + const db = c.env.DB_EXTENSIONS; + const publisher = getPublisher(c); + let ownerUserId = getAuth(c).userId; + // Moderator corrections can rebuild published source, but cannot upload new files for another developer. + if (form.source_id && !publisher) { + const correction = await db + .prepare( + ` + SELECT d.owner_user_id FROM package_sources s + JOIN developers d ON d.profile_generation=s.profile_generation + JOIN users moderator ON moderator.id=? AND moderator.is_moderator=1 AND moderator.deleted_at IS NULL + WHERE s.id=? AND s.extension_id=? AND s.version=? + AND EXISTS (SELECT 1 FROM package_artifacts a WHERE a.source_id=s.id AND a.published_at IS NOT NULL) + ` + ) + .bind(ownerUserId, form.source_id, manifest.id, manifest.version) + .first<{ owner_user_id: string }>(); + if (correction) ownerUserId = correction.owner_user_id; + } + const owner = await packageOwner( + db, + ownerUserId, + manifest.id, + publisher + ); + const baseUrl = getPlatform(c).getEnv("EXTENSION_PACKAGES_BASE_URL"); + if (!baseUrl) + throw new PackageError( + "PACKAGES_DISABLED", + "Package download URL is not configured", + 503 + ); + const jobs = new PackageJobsDatabase(db); + const source = form.source + ? new Uint8Array(await form.source.arrayBuffer()) + : undefined; + const uploadSha256 = source ? await sha256(source) : null; + const fingerprint = await sha256( + new Uint8Array( + new TextEncoder().encode( + canonicalJson({ + owner, + manifest, + source: uploadSha256 ?? form.source_id + }) + ) + ) + ); + let job = await jobs.find(fingerprint); + if (!job) { + const uploadObjectId = source + ? await reserveObject( + db, + store, + owner, + manifest.id, + source, + uploadSha256! + ) + : null; + job = await jobs.create({ + fingerprint, + owner, + manifest, + sourceId: form.source_id ?? null, + uploadObjectId, + uploadSha256 + }); + } + if (job.status === "queued") + await dispatchPackageJob( + jobs, + getPlatform(c).getQueue("EXTENSION_PACKAGE_QUEUE"), + job.id + ); + c.header( + "Location", + `${baseUrl.replace(/\/$/, "")}/package-jobs/${job.id}` + ); + c.header("Retry-After", "1"); + return c.json(await jobResponse(db, job), 202); + } catch (error) { + return failure(c, error); + } + } + ); + + app.openapi( + createRoute({ + method: "get", + path: "/package-jobs/{jobId}", + tags: ["Packages"], + summary: "Read an owning developer's private package-processing job", + security: [{ Bearer: [] }], + middleware: [requireContentWrite()] as const, + request: { params: PackageJobParamSchema }, + responses: { + 200: { + description: "Package processing state and completed artifact", + content: { + "application/json": { + schema: z.object({ result: PackageJobSchema }) + } + } + }, + ...packageErrors + } + }), + async (c) => { + try { + const db = c.env.DB_EXTENSIONS; + const job = await new PackageJobsDatabase(db).get( + c.req.valid("param").jobId + ); + if (!job) + throw new PackageError("NOT_FOUND", "Package job not found", 404); + const userId = getAuth(c).userId; + const moderator = await db + .prepare( + "SELECT 1 FROM users WHERE id=? AND is_moderator=1 AND deleted_at IS NULL" + ) + .bind(userId) + .first(); + if (!moderator) { + const publisher = getPublisher(c); + const owner = await packageOwner( + db, + userId, + job.manifest.id, + publisher + ); + if ( + owner.id !== job.owner.id || + owner.profile_generation !== job.owner.profile_generation || + owner.ownership_epoch !== job.owner.ownership_epoch || + (publisher && publisher.clientId !== job.owner.publisher_client_id) + ) + throw new PackageError("NOT_FOUND", "Package job not found", 404); + } + return c.json(await jobResponse(db, job), 200); + } catch (error) { + return failure(c, error); + } + } + ); + + // Registered before the parameter read. This only reclaims failed uploads; + // source/package rows and every published artifact remain immutable. + app.openapi( + createRoute({ + method: "post", + path: "/packages/cleanup", + tags: ["Packages"], + summary: "Reclaim abandoned local package objects older than one day", + security: [{ Bearer: [] }], + middleware: [requireModerator()] as const, + responses: { + 200: { + description: "Cleanup completed", + content: { + "application/json": { + schema: z.object({ result: z.object({ deleted: z.number() }) }) + } + } + }, + ...packageErrors + } + }), + async (c) => { + try { + await new PackageJobsDatabase(c.env.DB_EXTENSIONS).expire(); + return c.json( + { + result: { + deleted: await cleanupPackageObjects( + c.env.DB_EXTENSIONS, + packageStore(c)! + ) + } + }, + 200 + ); + } catch (error) { + return failure(c, error); + } + } + ); + + for (const resource of ["metadata", "download", "icon"] as const) { + const download = resource === "download"; + const icon = resource === "icon"; + app.openapi( + createRoute({ + method: "get", + path: icon + ? "/packages/{packageId}/icon" + : download + ? "/packages/{packageId}/download" + : "/packages/{packageId}", + tags: ["Packages"], + summary: download + ? "Download an approved package, or preview as owner/moderator" + : "Read an approved package or an authorized private preview", + security: [{ Bearer: [] }], + middleware: [optionalAuth()] as const, + request: { params: PackageParamSchema }, + responses: { + 200: + download || icon + ? { + description: icon + ? "Sanitized SVG, optimized AVIF/PNG, or generic fallback" + : "Verified immutable ZIP", + content: icon + ? { + "image/svg+xml": { + schema: z.string().openapi({ format: "binary" }) + }, + "image/png": { + schema: z.string().openapi({ format: "binary" }) + }, + "image/avif": { + schema: z.string().openapi({ format: "binary" }) + } + } + : { + "application/zip": { + schema: z.string().openapi({ format: "binary" }) + } + } + } + : { + description: "Package manifest and checksum", + content: { + "application/json": { + schema: z.object({ result: PackageArtifactSchema }) + } + } + }, + ...packageErrors + } + }), + async (c: Context<{ Bindings: CloudflareBindings }>) => { + try { + const auth = getOptionalAuth(c); + const row = await c.env.DB_EXTENSIONS.prepare( + `${ARTIFACT_SELECT} WHERE a.id=? AND ( + (a.published_at IS NOT NULL AND EXISTS (SELECT 1 FROM extensions e WHERE lower(e.id)=lower(s.extension_id) AND e.published_at IS NOT NULL AND e.delisted_at IS NULL)) + OR EXISTS (SELECT 1 FROM users u WHERE u.id=? AND u.deleted_at IS NULL AND (u.is_moderator=1 OR EXISTS (SELECT 1 FROM developers d WHERE d.profile_generation=s.profile_generation AND d.owner_user_id=u.id))) + )` + ) + .bind(c.req.param("packageId"), auth?.userId ?? "") + .first(); + if (!row) + throw new PackageError("NOT_FOUND", "Package not found", 404); + if (!download && !icon) return c.json(artifactResponse(row), 200); + if (icon) { + if (row.published_at) scheduleIconPublication(c, row.extension_id); + const avif = (c.req.header("Accept") ?? "") + .split(",") + .some((part) => { + const [type, ...params] = part.trim().split(";"); + return ( + type === "image/avif" && + !params.some((param) => /^\s*q=0(?:\.0*)?\s*$/.test(param)) + ); + }); + if (!row.icon_source_id) + throw new PackageError( + "ICON_UNAVAILABLE", + "Package icon has not been prepared", + 503 + ); + const key = + row.svg_object_id ?? + (avif ? row.avif_object_id : row.png_object_id); + const checksum = row.svg_object_id + ? row.svg_sha256 + : avif + ? row.avif_sha256 + : row.png_sha256; + const fallback = + ''; + const body = key + ? await packageStore(c)!.get(key, checksum!) + : fallback; + if (!body) + throw new PackageError( + "ICON_UNAVAILABLE", + "Stored icon is unavailable", + 503 + ); + + return new Response(body, { + headers: { + "Content-Type": row.svg_object_id + ? "image/svg+xml" + : key + ? avif + ? "image/avif" + : "image/png" + : "image/svg+xml", + "Content-Security-Policy": + "sandbox; default-src 'none'; style-src 'unsafe-inline'; img-src data:", + "X-Content-Type-Options": "nosniff", + "Cache-Control": "private, no-store", + Vary: "Authorization, Accept" + } + }); + } + const body = await packageStore(c)!.get(row.object_id, row.sha256); + if (!body) + throw new PackageError( + "PACKAGE_UNAVAILABLE", + "Package object is unavailable", + 503 + ); + return new Response(body, { + headers: { + "Content-Type": "application/zip", + "Content-Disposition": `attachment; filename="${row.extension_id}-${row.version}-${row.id}.zip"`, + ETag: `"${row.sha256}"`, + "X-Content-Type-Options": "nosniff", + "Cache-Control": "private, no-store", + Vary: "Authorization" + } + }); + } catch (error) { + return failure(c, error); + } + } + ); + } +} diff --git a/src/services/extensions/v2/schemas/badges.ts b/src/services/extensions/v2/schemas/badges.ts index 94a1f715..1d34628a 100644 --- a/src/services/extensions/v2/schemas/badges.ts +++ b/src/services/extensions/v2/schemas/badges.ts @@ -27,6 +27,7 @@ export const BadgeDataSchema = z.object({ releases: z .array( z.object({ + is_latest: z.boolean().optional(), tag: z .string() .min(1) diff --git a/src/services/extensions/v2/schemas/extensions.ts b/src/services/extensions/v2/schemas/extensions.ts index f40db85b..b678d480 100644 --- a/src/services/extensions/v2/schemas/extensions.ts +++ b/src/services/extensions/v2/schemas/extensions.ts @@ -16,6 +16,17 @@ export const ReleaseSchema = z date: z.string().min(1).max(64), download_url: httpUrl(), changelog_url: httpUrl().optional(), + is_latest: z + .boolean() + .optional() + .describe( + "Explicitly selected latest release; at most one release may be selected" + ), + artifact_id: z.uuid().optional(), + sha256: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), requires: ManifestRequirementsSchema }) .strict() @@ -50,7 +61,6 @@ export const ExtensionContentSchema = z releases: z.array(ReleaseSchema).min(1).max(100), homepage_url: ManifestSchema.shape.homepage_url, license: LicenseSchema, - icon_url: ManifestSchema.shape.icon_url, readme: z.string().min(1).max(100_000), source: RepositorySchema, version: ManifestSchema.shape.version, @@ -91,7 +101,7 @@ export const StoredExtensionContentSchema = ExtensionContentSchema.extend({ name: z.string().min(1).max(120), description: z.string().min(1).max(4000), version: z.string().min(1).max(100), - icon_url: z.union([ManifestSchema.shape.icon_url, httpUrl()]).optional(), + icon_url: z.string().max(2048).optional(), website: httpUrl().optional(), license: z.union([ LicenseSchema, diff --git a/src/services/extensions/v2/schemas/manifest.ts b/src/services/extensions/v2/schemas/manifest.ts index ebee1f46..e838dc6c 100644 --- a/src/services/extensions/v2/schemas/manifest.ts +++ b/src/services/extensions/v2/schemas/manifest.ts @@ -2,7 +2,7 @@ import { z } from "@hono/zod-openapi"; import { httpUrl } from "./common"; // Package metadata follows FOSSBilling/modules manifest.schema.json (v1). -// Package paths are kept relative; serving an icon is a publisher concern. +// Icons use fixed filenames at the package root, outside the manifest. export const MANIFEST_EXTENSION_TYPES = [ "module", "theme", @@ -59,12 +59,6 @@ export const ManifestSchema = z .max(1000) .regex(/^[A-Za-z0-9.\-+_:() |]+$/), homepage_url: httpUrl().optional(), - icon_url: z - .string() - .min(1) - .max(2048) - .regex(/^(?!\/)(?![a-zA-Z][a-zA-Z0-9+.-]*:)(?!.*\.\.)[^\\]+$/) - .optional(), requires: ManifestRequirementsSchema }) .strict() diff --git a/src/services/extensions/v2/schemas/packages.ts b/src/services/extensions/v2/schemas/packages.ts new file mode 100644 index 00000000..cd34e47d --- /dev/null +++ b/src/services/extensions/v2/schemas/packages.ts @@ -0,0 +1,52 @@ +import { z } from "@hono/zod-openapi"; +import { ManifestSchema } from "./manifest"; + +export const PackageBuildSchema = z + .object({ + manifest: z + .string() + .min(1) + .max(64 * 1024) + .describe("JSON manifest v1"), + source: z + .instanceof(File) + .optional() + .openapi({ type: "string", format: "binary" }), + source_id: z.uuid().optional() + }) + .strict() + .refine( + (value) => Boolean(value.source) !== Boolean(value.source_id), + "Supply either a source ZIP or a stored source_id" + ); +export const PackageParamSchema = z.object({ packageId: z.uuid() }); +export const PackageArtifactSchema = z + .object({ + id: z.uuid(), + source_id: z.uuid(), + manifest: ManifestSchema, + sha256: z.string().regex(/^[a-f0-9]{64}$/), + download_url: z.url(), + published_at: z.string().nullable(), + created_at: z.string() + }) + .openapi("PackageArtifact"); + +export const PackageJobParamSchema = z.object({ jobId: z.uuid() }); +export const PackageJobSchema = z + .object({ + id: z.uuid(), + extension_id: z.string(), + status: z.enum(["queued", "processing", "ready", "failed"]), + artifact: z.union([PackageArtifactSchema, z.null()]), + error: z + .object({ + code: z.string(), + message: z.string(), + status: z.number().int() + }) + .nullable(), + created_at: z.string(), + updated_at: z.string() + }) + .openapi("PackageJob"); diff --git a/test/lib/auth/bearer-assertion.test.ts b/test/lib/auth/bearer-assertion.test.ts index da218e21..44ef7c4d 100644 --- a/test/lib/auth/bearer-assertion.test.ts +++ b/test/lib/auth/bearer-assertion.test.ts @@ -20,6 +20,9 @@ function platformWithSecret( previousSecret?: string ): PlatformContext { return { + getQueue: () => { + throw new Error("No queue in auth test"); + }, getCache: () => { throw new Error("not implemented"); }, diff --git a/test/lib/releases.test.ts b/test/lib/releases.test.ts index 58deb2a3..650f43df 100644 --- a/test/lib/releases.test.ts +++ b/test/lib/releases.test.ts @@ -29,3 +29,18 @@ describe("sortReleasesDescending", () => { ]); }); }); + +// The same choice drives directory downloads and v1 version badges. +describe("latestRelease", () => { + it("honors a selected older release and otherwise prefers stable versions", async () => { + const { latestRelease } = await import("../../src/lib/releases"); + const older = { tag: "1.0.0", is_latest: true }; + expect(latestRelease([{ tag: "2.0.0" }, older])).toBe(older); + expect(latestRelease([{ tag: "3.0.0-rc.1" }, { tag: "2.0.0" }])?.tag).toBe( + "2.0.0" + ); + expect( + latestRelease([{ tag: "3.0.0-rc.1" }, { tag: "3.0.0-rc.2" }])?.tag + ).toBe("3.0.0-rc.2"); + }); +}); diff --git a/test/services/extensions/v2/b2-store.test.ts b/test/services/extensions/v2/b2-store.test.ts new file mode 100644 index 00000000..56932294 --- /dev/null +++ b/test/services/extensions/v2/b2-store.test.ts @@ -0,0 +1,158 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { b2PackageStore } from "../../../../src/services/extensions/v2/packages/b2-store"; +import { + sha256, + PACKAGE_LIMITS +} from "../../../../src/services/extensions/v2/package-builder"; + +const objectId = "134cf413-b5ab-4f57-b2ad-520020af880d"; +const config: Record = { + ENDPOINT: "https://s3.example.test", + REGION: "eu-central-003", + BUCKET: "private-packages", + ACCESS_KEY_ID: "test-key", + SECRET_ACCESS_KEY: "test-secret" +}; +const store = (overrides: Record = {}) => + b2PackageStore( + (name) => + ({ ...config, ...overrides })[name.replace("EXTENSION_PACKAGES_S3_", "")] + ); +afterEach(() => vi.unstubAllGlobals()); + +describe("private B2 package storage", () => { + it("signs ZIP writes and verifies downloaded bytes", async () => { + const bytes = new Uint8Array(new TextEncoder().encode("synthetic ZIP")); + const checksum = await sha256(bytes); + const requests: Request[] = []; + vi.stubGlobal("fetch", async (request: Request) => { + requests.push(request); + if (request.method === "PUT") { + expect(new Uint8Array(await request.arrayBuffer())).toEqual(bytes); + return new Response(null, { status: 200 }); + } + return new Response(bytes, { + headers: { "x-amz-meta-sha256": checksum } + }); + }); + const storage = store(); + await storage.put(objectId, bytes, checksum); + expect( + new Uint8Array( + await new Response(await storage.get(objectId, checksum)).arrayBuffer() + ) + ).toEqual(bytes); + expect(requests.map((r) => r.method)).toEqual(["PUT", "GET"]); + for (const request of requests) { + expect(request.url).toBe( + `https://s3.example.test/private-packages/local/packages/${objectId}` + ); + expect(request.headers.get("Authorization")).toMatch( + /^AWS4-HMAC-SHA256 / + ); + expect(request.redirect).toBe("manual"); + } + expect(requests[0].headers.get("Cache-Control")).toBe("private, no-store"); + expect(requests[0].headers.get("x-amz-content-sha256")).toBe(checksum); + }); + + it("rejects corrupted bodies even when checksum metadata matches", async () => { + const checksum = await sha256( + new Uint8Array(new TextEncoder().encode("expected")) + ); + vi.stubGlobal( + "fetch", + async () => + new Response("corrupted", { + headers: { "x-amz-meta-sha256": checksum } + }) + ); + await expect(store().get(objectId, checksum)).rejects.toThrow( + "checksum verification" + ); + }); + + it("bounds chunked downloads without trusting Content-Length", async () => { + vi.stubGlobal( + "fetch", + async () => + new Response( + new ReadableStream({ + start(controller) { + controller.enqueue( + new Uint8Array(PACKAGE_LIMITS.expandedBytes + 1024 * 1024 + 1) + ); + controller.close(); + } + }) + ) + ); + await expect(store().get(objectId)).rejects.toThrow("size limit"); + }); + + it("deletes the stored version rather than creating a delete marker", async () => { + const requests: Request[] = []; + vi.stubGlobal("fetch", async (request: Request) => { + requests.push(request); + return request.method === "HEAD" + ? new Response(null, { + headers: { "x-amz-version-id": "stored-version+1" } + }) + : new Response(null, { status: 204 }); + }); + await store().delete(objectId); + expect(requests.map((r) => r.method)).toEqual(["HEAD", "DELETE"]); + expect(new URL(requests[1].url).searchParams.get("versionId")).toBe( + "stored-version+1" + ); + }); + + it("treats missing reads and cleanup as absent, but fails closed on storage errors", async () => { + const fetch = vi + .fn() + .mockImplementation(async () => new Response(null, { status: 404 })); + vi.stubGlobal("fetch", fetch); + expect(await store().get(objectId)).toBeNull(); + await store().delete(objectId); + expect(fetch).toHaveBeenCalledTimes(2); + fetch.mockImplementation(async () => new Response(null, { status: 403 })); + await expect(store().get(objectId)).rejects.toThrow("unavailable"); + await expect(store().delete(objectId)).rejects.toThrow("cleanup"); + fetch.mockImplementation(async () => { + throw new Error("upstream timeout"); + }); + await expect(store().get(objectId)).rejects.toThrow("request failed"); + }); + + it("rejects failed uploads and retains cleanup failures for retry", async () => { + const bytes = new Uint8Array(new TextEncoder().encode("ZIP")); + vi.stubGlobal("fetch", async (request: Request) => + request.method === "HEAD" + ? new Response(null, { headers: { "x-amz-version-id": "v1" } }) + : new Response(null, { status: 503 }) + ); + await expect( + store().put(objectId, bytes, await sha256(bytes)) + ).rejects.toThrow("upload failed"); + await expect(store().delete(objectId)).rejects.toThrow("cleanup failed"); + }); + + it("rejects unsafe configuration, keys and mismatched upload checksums before network access", async () => { + const fetch = vi.fn(); + vi.stubGlobal("fetch", fetch); + for (const ENDPOINT of [ + "http://s3.example.test", + "https://key:secret@s3.example.test", + "https://s3.example.test/path", + "https://s3.example.test?key=value" + ]) + expect(() => store({ ENDPOINT })).toThrow("configuration"); + expect(() => store({ SECRET_ACCESS_KEY: undefined })).toThrow("incomplete"); + expect(() => store({ PREFIX: "../other" })).toThrow("configuration"); + await expect(store().get("../other")).rejects.toThrow("object ID"); + await expect( + store().put(objectId, new Uint8Array([1]), "wrong") + ).rejects.toThrow("checksum"); + expect(fetch).not.toHaveBeenCalled(); + }); +}); diff --git a/test/services/extensions/v2/badges.test.ts b/test/services/extensions/v2/badges.test.ts index 28454c3d..9a3a3811 100644 --- a/test/services/extensions/v2/badges.test.ts +++ b/test/services/extensions/v2/badges.test.ts @@ -44,6 +44,45 @@ beforeEach(async () => { }); describe("v2 public badges", () => { + it("uses the selected release for badges and the legacy version endpoint", async () => { + await db + .prepare("UPDATE extensions SET releases=? WHERE id='example'") + .bind( + JSON.stringify([ + { ...release("1.0.0", "^0.7"), is_latest: true }, + release("2.0.0", requirement) + ]) + ) + .run(); + expect(await (await badge("version")).text()).toContain( + "Latest version: v1.0.0" + ); + expect(await (await badge("fossbilling")).text()).toContain( + "FOSSBilling: ^0.7" + ); + expect(await (await get("/extensions/v1/example/version", {})).text()).toBe( + "1.0.0" + ); + }); + + it("prefers stable releases over newer prereleases when no release is selected", async () => { + await db + .prepare("UPDATE extensions SET releases=? WHERE id='example'") + .bind( + JSON.stringify([ + release("3.0.0-beta.1", "^1.0"), + release("2.0.0", requirement) + ]) + ) + .run(); + expect(await (await badge("version")).text()).toContain( + "Latest version: v2.0.0" + ); + expect(await (await get("/extensions/v1/example/version", {})).text()).toBe( + "2.0.0" + ); + }); + it("displays the complete latest-release requirement, including Composer exclusions", async () => { const response = await badge("fossbilling"); expect(response.status).toBe(200); diff --git a/test/services/extensions/v2/db-fixtures.ts b/test/services/extensions/v2/db-fixtures.ts index 9505f9b2..cc7398e7 100644 --- a/test/services/extensions/v2/db-fixtures.ts +++ b/test/services/extensions/v2/db-fixtures.ts @@ -133,6 +133,13 @@ async function clearDeveloperHistory(db: D1Database): Promise { } export async function resetExtensionsDb(db: D1Database): Promise { + await db.batch([ + db.prepare("DELETE FROM package_jobs"), + db.prepare("DELETE FROM package_artifacts"), + db.prepare("DELETE FROM package_icons"), + db.prepare("DELETE FROM package_sources"), + db.prepare("DELETE FROM package_objects") + ]); await db.prepare("DELETE FROM extension_write_events").run(); await db.prepare("DELETE FROM claim_verification_budgets").run(); await clearDeveloperHistory(db); diff --git a/test/services/extensions/v2/manifest.test.ts b/test/services/extensions/v2/manifest.test.ts index cfc35fa8..a4850efe 100644 --- a/test/services/extensions/v2/manifest.test.ts +++ b/test/services/extensions/v2/manifest.test.ts @@ -25,7 +25,6 @@ describe("manifest v1 publishing", () => { const content = { ...sampleCreate({ extensionId: "my_module" }), author: { name: "Package Author", url: "https://author.example.com" }, - icon_url: "assets/icon.svg", license: "MIT OR Apache-2.0", requires: { fossbilling: "^0.8.9 || ^1.0", diff --git a/test/services/extensions/v2/package-builder.test.ts b/test/services/extensions/v2/package-builder.test.ts new file mode 100644 index 00000000..74a62847 --- /dev/null +++ b/test/services/extensions/v2/package-builder.test.ts @@ -0,0 +1,323 @@ +import { describe, expect, it } from "vitest"; +import { + TextReader, + TextWriter, + Uint8ArrayReader, + Uint8ArrayWriter, + ZipReader, + ZipWriter +} from "@zip.js/zip.js"; +import { + buildExtensionPackage, + PACKAGE_LIMITS, + sha256 +} from "../../../../src/services/extensions/v2/package-builder"; + +const manifest = { + manifest_version: 1, + type: "module", + id: "demo", + name: "Demo", + version: "1.2.0", + description: "Example extension", + author: { name: "Example Developer" }, + license: "Apache-2.0", + requires: { fossbilling: ">=0.8.0" } +}; +const options = { useWebWorkers: false, useCompressionStream: true }; +async function zip( + files: [string, string][], + entryOptions: Parameters>["add"]>[2] = {} +) { + const writer = new ZipWriter(new Uint8ArrayWriter(), options); + for (const [name, content] of files) { + await writer.add(name, new TextReader(content), entryOptions); + } + return writer.close(); +} +async function contents(bytes: Uint8Array) { + const reader = new ZipReader(new Uint8ArrayReader(bytes), options); + try { + const result: Record = {}; + for await (const entry of reader.getEntriesGenerator()) { + if (!entry.directory) + result[entry.filename] = await entry.getData(new TextWriter()); + } + return result; + } finally { + await reader.close(); + } +} + +describe("local directory package builder", () => { + it("replaces the manifest and preserves every source file", async () => { + const source = await zip([ + ["manifest.json", "old metadata"], + ["Service.php", ""] + ]); + const built = await buildExtensionPackage(source, manifest); + expect(await contents(built.bytes)).toEqual({ + "manifest.json": JSON.stringify(manifest, null, 2) + "\n", + "Service.php": "" + }); + expect(built.sourceFileCount).toBe(2); + expect(built.sourceSha256).toBe(await sha256(source)); + expect(built.packageSha256).toBe(await sha256(built.bytes)); + const repeated = await buildExtensionPackage(source, manifest); + expect(repeated.packageSha256).toBe(built.packageSha256); + const correction = await buildExtensionPackage(source, { + ...manifest, + description: "Corrected description" + }); + expect(correction.sourceSha256).toBe(built.sourceSha256); + expect(correction.manifest.version).toBe(built.manifest.version); + expect(correction.packageSha256).not.toBe(built.packageSha256); + }); + + it.each([0, 8])( + "preserves the exact compressed payload for method %i", + async (compressionMethod) => { + const source = await zip( + [["file.php", ") { + const reader = new ZipReader(new Uint8ArrayReader(bytes), options); + try { + const entries = await reader.getEntries(); + const entry = entries.find((entry) => entry.filename === "file.php")!; + if (!entry || entry.directory) + throw new Error("Expected a regular file"); + return { + method: entry.compressionMethod, + bytes: await entry.getData(new Uint8ArrayWriter(), { + passThrough: true + }) + }; + } finally { + await reader.close(); + } + } + const original = await payload(source); + expect(original.method).toBe(compressionMethod); + expect(await payload(built.bytes)).toEqual(original); + expect((await contents(built.bytes))["file.php"]).toBe( + " { + const source = await zip([ + [`icon.${format}`, "image bytes"], + ["file.php", " { + const source = await zip([ + ["icon.svg", "svg"], + ["icon.png", "png"] + ]); + await expect(buildExtensionPackage(source, manifest)).rejects.toThrow( + "only one" + ); + await expect( + buildExtensionPackage(await zip([["file.php", " { + const built = await buildExtensionPackage( + await zip([["assets/icon.svg", "image"]]), + manifest + ); + expect(built.icon).toBeNull(); + }); + + it("inserts a missing manifest and rejects invalid metadata", async () => { + const source = await zip([["file.php", " { + await expect( + buildExtensionPackage(await zip([[path, "test"]]), manifest) + ).rejects.toThrow(/Unsafe/); + }); + + it("rejects case collisions, file/directory conflicts and symlinks", async () => { + await expect( + buildExtensionPackage( + await zip([ + ["A.php", "a"], + ["a.php", "b"] + ]), + manifest + ) + ).rejects.toThrow("Duplicate package path"); + await expect( + buildExtensionPackage( + await zip([ + ["dir", "a"], + ["dir/file.php", "b"] + ]), + manifest + ) + ).rejects.toThrow("File conflicts"); + await expect( + buildExtensionPackage( + await zip([["link", "../outside"]], { unixMode: 0o120777 }), + manifest + ) + ).rejects.toThrow("Unsupported package entry"); + }); + + it("rejects files beneath the reserved manifest path", async () => { + await expect( + buildExtensionPackage( + await zip([["manifest.json/file.php", "bad"]]), + manifest + ) + ).rejects.toThrow("Manifest path is a directory"); + }); + + it("rejects encrypted files and truncated archives", async () => { + await expect( + buildExtensionPackage( + await zip([["file", "secret"]], { password: "test" }), + manifest + ) + ).rejects.toThrow("Unsupported package entry"); + const source = await zip([["file", "content"]]); + await expect( + buildExtensionPackage(source.slice(0, -30), manifest) + ).rejects.toThrow(); + }); + + it.each(["manifest.json", "file.php"])( + "checks CRCs before copying or replacing %s", + async (filename) => { + const source = await zip([[filename, "unique-contents"]], { + level: 0 + }); + const offset = new TextDecoder() + .decode(source) + .indexOf("unique-contents"); + expect(offset).toBeGreaterThan(0); + source[offset] ^= 1; + await expect(buildExtensionPackage(source, manifest)).rejects.toThrow(); + } + ); + + it.each([-1, 1])( + "rejects inconsistent uncompressed sizes (%i)", + async (difference) => { + const source = await zip([["file.php", "unique-contents"]]); + const view = new DataView( + source.buffer, + source.byteOffset, + source.byteLength + ); + view.setUint32(22, "unique-contents".length + difference, true); + for (let offset = 0; offset < source.length - 46; offset++) { + if (view.getUint32(offset, true) === 0x08074b50) { + view.setUint32( + offset + 12, + "unique-contents".length + difference, + true + ); + } + if (view.getUint32(offset, true) === 0x02014b50) { + view.setUint32( + offset + 24, + "unique-contents".length + difference, + true + ); + break; + } + } + await expect(buildExtensionPackage(source, manifest)).rejects.toThrow(); + } + ); + + it("rejects oversized inputs and expansion bombs", async () => { + await expect( + buildExtensionPackage( + new Uint8Array(PACKAGE_LIMITS.sourceBytes + 1), + manifest + ) + ).rejects.toThrow("Source ZIP exceeds size limit"); + const source = await zip([ + ["large.txt", "x".repeat(PACKAGE_LIMITS.expandedBytes + 1)] + ]); + expect(source.byteLength).toBeLessThan(PACKAGE_LIMITS.sourceBytes); + await expect(buildExtensionPackage(source, manifest)).rejects.toThrow( + "expanded size limit" + ); + }); + + it("rejects entry floods and manifest-only archives", async () => { + const source = await zip( + Array.from({ length: PACKAGE_LIMITS.entries + 1 }, (_, i) => [ + `file-${i}`, + "" + ]) + ); + await expect(buildExtensionPackage(source, manifest)).rejects.toThrow( + "entry limit" + ); + await expect( + buildExtensionPackage(await zip([["manifest.json", "{}"]]), manifest) + ).rejects.toThrow("no source files"); + }); + + it("builds and reads back a representative multi-file package in Workers", async () => { + const source = await zip( + Array.from({ length: 100 }, (_, i) => [ + `src/file-${i}.php`, + " ({ ...job }), + claim: async (_id, lease) => { + if (job.status !== "queued") return null; + activeLease = lease; + job.status = "processing"; + job.attempts++; + return { ...job }; + }, + complete: async (_id, lease, artifactId) => { + if (lease === activeLease) { + job.status = "ready"; + job.artifactId = artifactId; + activeLease = null; + } + }, + fail: async (_id, lease, error, retry) => { + if (lease === activeLease) { + job.status = retry ? "queued" : "failed"; + job.error = error; + activeLease = null; + } + }, + pending: async () => [job.id], + markEnqueued: vi.fn().mockResolvedValue(undefined) + }; + return { job, jobs }; +} + +describe("provider-independent package job processing", () => { + it("publishes only an identifier through the queue interface", async () => { + const { jobs, job } = repository(); + const send = vi.fn().mockResolvedValue(undefined); + await dispatchPackageJob(jobs, { send }, job.id); + expect(send).toHaveBeenCalledWith({ jobId: job.id }); + expect(jobs.markEnqueued).toHaveBeenCalledWith(job.id); + }); + it("does not run completed jobs twice", async () => { + const { jobs, job } = repository(); + const prepare = vi.fn().mockResolvedValue("artifact"); + expect(await processPackageJob(jobs, job.id, prepare)).toBe("ack"); + expect(await processPackageJob(jobs, job.id, prepare)).toBe("ack"); + expect(prepare).toHaveBeenCalledTimes(1); + expect(job.artifactId).toBe("artifact"); + }); + it("retries transient failures with a bounded attempt count", async () => { + const { jobs, job } = repository(); + const prepare = vi.fn().mockRejectedValue(new Error("storage unavailable")); + expect(await processPackageJob(jobs, job.id, prepare)).toBe("retry"); + expect(await processPackageJob(jobs, job.id, prepare)).toBe("retry"); + expect(await processPackageJob(jobs, job.id, prepare)).toBe("ack"); + expect(job.status).toBe("failed"); + expect(job.error?.message).not.toContain("storage unavailable"); + }); + it("does not retry invalid packages or revoked ownership", async () => { + for (const status of [422, 403] as const) { + const { jobs, job } = repository(); + expect( + await processPackageJob(jobs, job.id, async () => { + throw new PackageError("REJECTED", "Rejected", status); + }) + ).toBe("ack"); + expect(job.status).toBe("failed"); + expect(job.attempts).toBe(1); + } + }); +}); diff --git a/test/services/extensions/v2/packages.test.ts b/test/services/extensions/v2/packages.test.ts new file mode 100644 index 00000000..d416ee14 --- /dev/null +++ b/test/services/extensions/v2/packages.test.ts @@ -0,0 +1,1467 @@ +import { PackageJobsDatabase } from "../../../../src/services/extensions/v2/db/package-jobs"; +import { processPackageJob } from "../../../../src/services/extensions/v2/packages/jobs"; +import { prepareQueuedPackage } from "../../../../src/services/extensions/v2/packages/cloudflare-queue"; +import { beforeEach, afterEach, describe, expect, it, vi } from "vitest"; +import { env } from "cloudflare:workers"; +import { + createExecutionContext, + waitOnExecutionContext +} from "cloudflare:test"; +import { + TextReader, + Uint8ArrayReader, + Uint8ArrayWriter, + ZipWriter, + ZipReader, + TextWriter +} from "@zip.js/zip.js"; +import app from "../../../../src/app"; +import { + signAccessToken, + activateM2mJwks, + deactivateM2mJwks +} from "../../../lib/auth/m2m-helper"; +import { PackageArtifactSchema } from "../../../../src/services/extensions/v2/schemas/packages"; +import { ManifestSchema } from "../../../../src/services/extensions/v2/schemas/manifest"; +import { ExtensionContentSchema } from "../../../../src/services/extensions/v2/schemas/extensions"; +import { contentManifest } from "../../../../src/services/extensions/v2/db/packages"; +import { sha256 } from "../../../../src/services/extensions/v2/package-builder"; +import { + setupExtensionsV2Tests, + authHeaders, + seedDeveloper, + sampleContent, + post, + get, + put, + db +} from "./harness"; +import { insertUser, getExtension } from "./db-fixtures"; + +vi.mock("@octokit/request", async () => + (await import("../../../mocks/octokit")).octokitRequestMock() +); +setupExtensionsV2Tests(); +const options = { useWebWorkers: false, useCompressionStream: true }; +const root = "/extensions/v2"; +const manifest = () => + contentManifest( + "package-demo", + ExtensionContentSchema.parse(sampleContent()) + ); +const bucket = () => env.EXTENSION_PACKAGE_BUCKET!; + +beforeEach(async () => { + vi.spyOn(env.EXTENSION_PACKAGE_QUEUE, "send").mockResolvedValue({ + metadata: { metrics: { backlogCount: 0, backlogBytes: 0 } } + }); + delete env.EXTENSION_PACKAGES_S3_ENDPOINT; + delete env.EXTENSION_ASSETS_S3_ENDPOINT; + env.ENVIRONMENT = "local"; + env.EXTENSION_PACKAGES_MODE = "local"; + env.EXTENSION_PACKAGES_BASE_URL = "http://localhost:8789/extensions/v2"; + await authHeaders("owner"); + await seedDeveloper("developer", "owner"); +}); +afterEach(() => { + vi.restoreAllMocks(); + vi.unstubAllGlobals(); + for (const key of [ + "S3_ENDPOINT", + "S3_REGION", + "S3_BUCKET", + "S3_ACCESS_KEY_ID", + "S3_SECRET_ACCESS_KEY", + "PUBLIC_URL", + "S3_PREFIX" + ]) + delete (env as unknown as Record)[ + `EXTENSION_ASSETS_${key}` + ]; + for (const suffix of [ + "ENDPOINT", + "REGION", + "BUCKET", + "ACCESS_KEY_ID", + "SECRET_ACCESS_KEY", + "PREFIX" + ]) + delete (env as unknown as Record)[ + `EXTENSION_PACKAGES_S3_${suffix}` + ]; + env.EXTENSION_PACKAGES_MODE = "disabled"; +}); +async function zip(content = ", + sourceId?: string, + user = "owner", + process = true +) { + const headers = await authHeaders(user); + delete headers["Content-Type"]; + const form = new FormData(); + form.set("manifest", JSON.stringify(metadata)); + if (sourceId) form.set("source_id", sourceId); + else + form.set( + "source", + new File([source ?? (await zip())], "source.zip", { + type: "application/zip" + }) + ); + const ctx = createExecutionContext(); + const response = await app.request( + `${root}/packages`, + { method: "POST", headers, body: form }, + env, + ctx + ); + await waitOnExecutionContext(ctx); + return process ? finishJob(response) : response; +} +async function finishJob(response: Response): Promise { + if (response.status !== 202) return response; + const { result } = (await response.json()) as { result: { id: string } }; + const jobs = new PackageJobsDatabase(env.DB_EXTENSIONS); + let job = (await jobs.get(result.id))!; + for ( + let attempt = 0; + attempt < 100 && job.status !== "ready" && job.status !== "failed"; + attempt++ + ) { + await processPackageJob(jobs, result.id, (job) => + prepareQueuedPackage(env, job) + ); + job = (await jobs.get(result.id))!; + if (job.status === "processing") + await new Promise((resolve) => setTimeout(resolve, 5)); + } + if (job.status === "ready") { + const read = await get( + `${root}/packages/${job.artifactId}`, + await authHeaders("owner") + ); + return new Response(await read.text(), { + status: 201, + headers: { "Content-Type": "application/json" } + }); + } + return Response.json( + { error: job.error }, + { status: job.error?.status ?? 503 } + ); +} +async function built( + metadata: unknown = manifest(), + source?: Uint8Array, + sourceId?: string +) { + const response = await upload(metadata, source, sourceId); + const body = (await response.json()) as { result: unknown }; + expect(response.status, JSON.stringify(body)).toBe(201); + return PackageArtifactSchema.parse(body.result); +} +function content(artifact: ReturnType) { + const { id: _id, ...metadata } = artifact.manifest; + return { + ...sampleContent(), + ...metadata, + download_url: artifact.download_url, + releases: [ + { + tag: artifact.manifest.version, + date: "2026-11-05", + requires: artifact.manifest.requires, + artifact_id: artifact.id, + sha256: artifact.sha256, + download_url: artifact.download_url + } + ] + }; +} +async function submitted( + artifact: ReturnType +) { + const response = await post( + `${root}/extensions`, + await authHeaders("owner"), + { id: "package-demo", ...content(artifact) } + ); + const body = (await response.json()) as { result: { revision_id: string } }; + expect(response.status, JSON.stringify(body)).toBe(201); + return body.result.revision_id; +} +async function approve(revision: string) { + await insertUser(db, { id: "moderator", is_moderator: 1 }); + return post( + `${root}/extensions/package-demo/revisions/${revision}/approve`, + await authHeaders("moderator"), + {} + ); +} +async function unpack(bytes: Uint8Array) { + const reader = new ZipReader(new Uint8ArrayReader(bytes), options); + const result: Record = {}; + try { + for await (const entry of reader.getEntriesGenerator()) + if (!entry.directory) + result[entry.filename] = await entry.getData(new TextWriter()); + } finally { + await reader.close(); + } + return result; +} + +describe("publisher package builds", () => { + beforeEach(async () => { + await activateM2mJwks(); + await insertUser(db, { id: "publisher-moderator", is_moderator: 1 }); + const response = await put( + `${root}/m2m/clients/package-publisher`, + await authHeaders("publisher-moderator"), + { developer_id: "developer", can_publish: true } + ); + expect(response.status).toBe(200); + }); + afterEach(() => deactivateM2mJwks()); + async function machineBuild(scopes = ["extensions:push"], process = true) { + const form = new FormData(); + form.set("manifest", JSON.stringify(manifest())); + form.set("source", new File([await zip()], "source.zip")); + const response = await app.request( + `${root}/packages`, + { + method: "POST", + headers: { + Authorization: `Bearer ${await signAccessToken({ clientId: "package-publisher", scopes })}` + }, + body: form + }, + env + ); + return process ? finishJob(response) : response; + } + it("lets an active machine publisher poll its own queued and ready job", async () => { + const response = await machineBuild(["extensions:push"], false); + expect(response.status).toBe(202); + const { result: job } = (await response.json()) as { + result: { id: string }; + }; + const headers = { + Authorization: `Bearer ${await signAccessToken({ clientId: "package-publisher", scopes: ["extensions:push"] })}` + }; + const pending = await get(`${root}/package-jobs/${job.id}`, headers); + expect(pending.status).toBe(200); + expect(await pending.json()).toMatchObject({ + result: { status: "queued", artifact: null } + }); + expect( + (await finishJob(Response.json({ result: job }, { status: 202 }))).status + ).toBe(201); + const ready = await get(`${root}/package-jobs/${job.id}`, headers); + expect(ready.status).toBe(200); + expect(await ready.json()).toMatchObject({ + result: { + status: "ready", + artifact: { manifest: { id: "package-demo" } } + } + }); + await db + .prepare( + "UPDATE m2m_publisher_clients SET revoked_at=CURRENT_TIMESTAMP WHERE client_id='package-publisher'" + ) + .run(); + expect((await get(`${root}/package-jobs/${job.id}`, headers)).status).toBe( + 403 + ); + }); + it("builds under the mapped developer while retaining owner privacy", async () => { + const response = await machineBuild(); + expect(response.status).toBe(201); + const artifact = PackageArtifactSchema.parse( + ((await response.json()) as { result: unknown }).result + ); + const source = await db + .prepare("SELECT profile_generation FROM package_sources WHERE id=?") + .bind(artifact.source_id) + .first(); + expect(source).toBeTruthy(); + expect( + (await get(`${root}/packages/${artifact.id}/download`, {})).status + ).toBe(404); + expect( + ( + await get( + `${root}/packages/${artifact.id}/download`, + await authHeaders("owner") + ) + ).status + ).toBe(200); + const token = await signAccessToken({ + clientId: "package-publisher", + scopes: ["extensions:push", "extensions:publish"] + }); + const headers = { + Authorization: `Bearer ${token}`, + "Content-Type": "application/json" + }; + const created = await post(`${root}/extensions`, headers, { + id: "package-demo", + ...content(artifact) + }); + expect(created.status).toBe(201); + const revision = ( + (await created.json()) as { result: { revision_id: string } } + ).result.revision_id; + expect( + ( + await post( + `${root}/extensions/package-demo/revisions/${revision}/approve`, + headers, + {} + ) + ).status + ).toBe(200); + expect( + (await get(`${root}/packages/${artifact.id}/download`, {})).status + ).toBe(200); + }); + it("requires push scope and refuses revoked or stale bindings", async () => { + expect((await machineBuild(["extensions:publish"])).status).toBe(403); + await db + .prepare( + "UPDATE m2m_publisher_clients SET revoked_at=CURRENT_TIMESTAMP WHERE client_id='package-publisher'" + ) + .run(); + expect((await machineBuild()).status).toBe(403); + await db + .prepare( + "UPDATE m2m_publisher_clients SET revoked_at=NULL,ownership_epoch=ownership_epoch+1 WHERE client_id='package-publisher'" + ) + .run(); + expect((await machineBuild()).status).toBe(403); + }); + it("does not reserve a package belonging to a different developer", async () => { + await seedDeveloper("another-developer", "another-owner"); + await db + .prepare( + "UPDATE m2m_publisher_clients SET developer_id='another-developer' WHERE client_id='package-publisher'" + ) + .run(); + // New IDs can be reserved by that developer, but an existing owned ID cannot. + const artifact = await built(); + await submitted(artifact); + expect((await machineBuild()).status).toBe(403); + }); + it("rechecks revocation after uploading before linking source bytes", async () => { + const original = bucket().put.bind(bucket()); + const write = vi + .spyOn(bucket(), "put") + .mockImplementationOnce(async (...args) => { + const result = await original(...args); + await db + .prepare( + "UPDATE m2m_publisher_clients SET revoked_at=CURRENT_TIMESTAMP WHERE client_id='package-publisher'" + ) + .run(); + return result; + }); + try { + expect((await machineBuild()).status).toBe(403); + expect( + await db + .prepare("SELECT count(*) as count FROM package_sources") + .first() + ).toEqual({ count: 0 }); + } finally { + write.mockRestore(); + } + }); +}); + +describe("queued package preparation", () => { + it("accepts a private job without preparing an artifact and deduplicates retries", async () => { + const source = await zip(); + const response = await upload( + manifest(), + source, + undefined, + "owner", + false + ); + expect(response.status).toBe(202); + const { result: job } = (await response.json()) as { + result: { id: string; status: string }; + }; + expect(job.status).toBe("queued"); + expect( + await db + .prepare("SELECT count(*) AS count FROM package_artifacts") + .first() + ).toEqual({ count: 0 }); + expect(env.EXTENSION_PACKAGE_QUEUE.send).toHaveBeenCalledWith( + { jobId: job.id }, + { contentType: "json" } + ); + const repeat = await upload(manifest(), source, undefined, "owner", false); + expect( + ((await repeat.json()) as { result: { id: string } }).result.id + ).toBe(job.id); + expect((await get(`${root}/package-jobs/${job.id}`, {})).status).toBe(401); + expect( + ( + await get( + `${root}/package-jobs/${job.id}`, + await authHeaders("stranger") + ) + ).status + ).toBe(403); + const finished = await finishJob( + Response.json({ result: job }, { status: 202 }) + ); + expect(finished.status).toBe(201); + const status = await get( + `${root}/package-jobs/${job.id}`, + await authHeaders("owner") + ); + expect( + ((await status.json()) as { result: { status: string } }).result.status + ).toBe("ready"); + }); + it("does not process a job after its developer ownership changes", async () => { + const response = await upload( + manifest(), + undefined, + undefined, + "owner", + false + ); + const { result: job } = (await response.json()) as { + result: { id: string }; + }; + await db + .prepare( + "UPDATE developers SET ownership_epoch=ownership_epoch+1 WHERE id='developer'" + ) + .run(); + expect( + (await finishJob(Response.json({ result: job }, { status: 202 }))).status + ).toBe(403); + expect( + await db + .prepare("SELECT count(*) AS count FROM package_artifacts") + .first() + ).toEqual({ count: 0 }); + }); + it("keeps the job recoverable if queue publication fails", async () => { + vi.mocked(env.EXTENSION_PACKAGE_QUEUE.send).mockRejectedValueOnce( + new Error("queue unavailable") + ); + expect( + (await upload(manifest(), undefined, undefined, "owner", false)).status + ).toBe(503); + const jobs = new PackageJobsDatabase(db); + const ids = await jobs.pending(); + expect(ids).toHaveLength(1); + const { dispatchPackageJob } = + await import("../../../../src/services/extensions/v2/packages/jobs"); + const send = vi.fn().mockResolvedValue(undefined); + await dispatchPackageJob(jobs, { send }, ids[0]); + expect(send).toHaveBeenCalledWith({ jobId: ids[0] }); + expect(await jobs.pending()).toEqual([]); + }); + it("recovers an expired processing lease without accepting stale completion", async () => { + const response = await upload( + manifest(), + undefined, + undefined, + "owner", + false + ); + const { result: job } = (await response.json()) as { + result: { id: string }; + }; + const jobs = new PackageJobsDatabase(db); + expect(await jobs.claim(job.id, "first")).not.toBeNull(); + expect(await jobs.claim(job.id, "second")).toBeNull(); + await db + .prepare( + "UPDATE package_jobs SET lease_until=datetime('now','-1 minute') WHERE id=?" + ) + .bind(job.id) + .run(); + const claimed = await jobs.claim(job.id, "second"); + expect(claimed?.attempts).toBe(2); + await jobs.fail( + job.id, + "first", + { code: "STALE", message: "Old worker", status: 503 }, + false + ); + expect((await jobs.get(job.id))!.status).toBe("processing"); + const artifactId = await prepareQueuedPackage(env, claimed!); + await jobs.complete(job.id, "second", artifactId); + expect((await jobs.get(job.id))!.status).toBe("ready"); + }); + it("claims a job once across concurrent deliveries", async () => { + const response = await upload( + manifest(), + undefined, + undefined, + "owner", + false + ); + const { result: job } = (await response.json()) as { + result: { id: string }; + }; + const jobs = new PackageJobsDatabase(db); + const prepare = vi.fn( + ( + job: import("../../../../src/services/extensions/v2/packages/jobs").PackageJob + ) => prepareQueuedPackage(env, job) + ); + await Promise.all([ + processPackageJob(jobs, job.id, prepare), + processPackageJob(jobs, job.id, prepare) + ]); + expect(prepare).toHaveBeenCalledTimes(1); + expect((await jobs.get(job.id))!.status).toBe("ready"); + }); +}); + +describe("stored directory packages", () => { + it("uses private B2 ZIP storage through build, rebuild and approval without publishing pending images", async () => { + Object.assign(env, { + EXTENSION_PACKAGES_S3_ENDPOINT: "https://s3.example.test", + EXTENSION_PACKAGES_S3_REGION: "eu-central-003", + EXTENSION_PACKAGES_S3_BUCKET: "test-packages", + EXTENSION_PACKAGES_S3_ACCESS_KEY_ID: "test-key", + EXTENSION_PACKAGES_S3_SECRET_ACCESS_KEY: "test-secret" + }); + const objects = new Map(); + const requests: Request[] = []; + vi.stubGlobal("fetch", async (request: Request) => { + requests.push(request); + expect(request.headers.get("Authorization")).toMatch( + /^AWS4-HMAC-SHA256 / + ); + const key = new URL(request.url).pathname; + expect(key).toMatch(/^\/test-packages\/local\/packages\/[a-f0-9-]+$/); + if (request.method === "PUT") { + objects.set(key, { + bytes: await request.arrayBuffer(), + checksum: request.headers.get("x-amz-meta-sha256")! + }); + return new Response(null); + } + const stored = objects.get(key); + return stored + ? new Response(stored.bytes, { + headers: { "x-amz-meta-sha256": stored.checksum } + }) + : new Response(null, { status: 404 }); + }); + const writer = new ZipWriter(new Uint8ArrayWriter(), options); + await writer.add("Service.php", new TextReader("' + ) + ); + const first = await built(manifest(), await writer.close()); + const revised = await built( + { ...manifest(), description: "Rebuilt from B2 source" }, + undefined, + first.source_id + ); + expect(revised.source_id).toBe(first.source_id); + expect(objects.size).toBe(3); + const path = `${root}/packages/${revised.id}/download`; + const before = requests.length; + expect((await get(path, {})).status).toBe(404); + expect(requests.length).toBe(before); + const owner = await get(path, await authHeaders("owner")); + expect(owner.status).toBe(200); + expect(await sha256(new Uint8Array(await owner.arrayBuffer()))).toBe( + revised.sha256 + ); + expect((await get(`${root}/packages/${revised.id}/icon`, {})).status).toBe( + 404 + ); + expect( + ( + await get( + `${root}/packages/${revised.id}/icon`, + await authHeaders("owner") + ) + ).status + ).toBe(200); + const revision = await submitted(revised); + expect((await approve(revision)).status).toBe(200); + const publicDownload = await get(path, {}); + expect(publicDownload.status).toBe(200); + expect(publicDownload.headers.get("Cache-Control")).toBe( + "private, no-store" + ); + expect( + await sha256(new Uint8Array(await publicDownload.arrayBuffer())) + ).toBe(revised.sha256); + }); + + it("keeps uploads private until approval and publishes the exact reviewed ZIP", async () => { + const artifact = await built(); + const path = `${root}/packages/${artifact.id}/download`; + expect((await get(path, {})).status).toBe(404); + expect((await get(path, await authHeaders("stranger"))).status).toBe(404); + expect((await get(path, await authHeaders("owner"))).status).toBe(200); + const revision = await submitted(artifact); + expect((await get(path, {})).status).toBe(404); + expect((await approve(revision)).status).toBe(200); + const download = await get(path, {}); + expect(download.status).toBe(200); + expect(download.headers.get("Cache-Control")).toContain("no-store"); + const bytes = new Uint8Array(await download.arrayBuffer()); + expect(await sha256(bytes)).toBe(artifact.sha256); + const files = await unpack(bytes); + expect(files["Service.php"]).toBe(" { + const writer = new ZipWriter(new Uint8ArrayWriter(), options); + await writer.add("Service.php", new TextReader("' + ) + ); + const artifact = await built(manifest(), await writer.close()); + const path = `${root}/packages/${artifact.id}/icon`; + expect((await get(path, {})).status).toBe(404); + expect((await get(path, await authHeaders("stranger"))).status).toBe(404); + const preview = await get(path, await authHeaders("owner")); + expect(preview.status).toBe(200); + expect(preview.headers.get("Content-Type")).toBe("image/svg+xml"); + expect(preview.headers.get("Content-Security-Policy")).toContain("sandbox"); + expect(await preview.text()).toContain("(); + await bucket().delete(object!.object_id); + // Reading the icon must not depend on opening the ZIP. + expect((await get(path, await authHeaders("owner"))).status).toBe(200); + expect((await approve(await submitted(artifact))).status).toBe(200); + expect((await get(path, {})).status).toBe(200); + const published = await getExtension(db, "package-demo"); + expect(published?.icon_url).toBe( + artifact.download_url.replace(/\/download$/, "/icon") + ); + await db + .prepare( + "UPDATE extensions SET delisted_at=CURRENT_TIMESTAMP WHERE id='package-demo'" + ) + .run(); + expect((await get(path, {})).status).toBe(404); + }); + it("sanitizes SVG scripts, events, foreign content, and stylesheets while preserving vector artwork", async () => { + const original = ` + +
HTML
 + + + + `; + const writer = new ZipWriter(new Uint8ArrayWriter(), options); + await writer.add("icon.svg", new TextReader(original)); + const artifact = await built(manifest(), await writer.close()); + const response = await get( + `${root}/packages/${artifact.id}/icon`, + await authHeaders("owner") + ); + const svg = await response.text(); + expect(response.headers.get("Content-Type")).toBe("image/svg+xml"); + expect(response.headers.get("Content-Security-Policy")).toBe( + "sandbox; default-src 'none'; style-src 'unsafe-inline'; img-src data:" + ); + expect(response.headers.get("X-Content-Type-Options")).toBe("nosniff"); + expect(svg).not.toMatch( + /script|onload|foreignObject|untrusted\.example|xml-stylesheet/ + ); + expect(svg).toContain("linearGradient"); + expect(svg).toContain("url(#paint)"); + expect(svg).toContain('viewBox="0 0 24 24"'); + const row = await db + .prepare( + "SELECT svg_object_id,avif_object_id,png_object_id FROM package_icons WHERE source_id=?" + ) + .bind(artifact.source_id) + .first<{ + svg_object_id: string; + avif_object_id: null; + png_object_id: null; + }>(); + expect(row?.svg_object_id).toMatch(/^icons\/.+\.svg$/); + expect(row?.avif_object_id).toBeNull(); + expect(row?.png_object_id).toBeNull(); + const download = await get( + `${root}/packages/${artifact.id}/download`, + await authHeaders("owner") + ); + const zip = new ZipReader( + new Uint8ArrayReader(new Uint8Array(await download.arrayBuffer())), + options + ); + try { + const icon = (await zip.getEntries()).find( + (entry) => entry.filename === "icon.svg" + )!; + if (icon.directory) throw new Error("Expected icon file"); + expect(await icon.getData(new TextWriter())).toBe(original); + } finally { + await zip.close(); + } + }); + it.each([ + "', + '" + ])("rejects malformed, non-SVG, and oversized SVG files %#", async (svg) => { + const writer = new ZipWriter(new Uint8ArrayWriter(), options); + await writer.add("icon.svg", new TextReader(svg)); + expect((await upload(manifest(), await writer.close())).status).toBe(422); + expect( + await db.prepare("SELECT count(*) AS count FROM package_sources").first() + ).toEqual({ count: 0 }); + }); + it("sanitizes SVG content even with a raster filename", async () => { + const writer = new ZipWriter(new Uint8ArrayWriter(), options); + await writer.add( + "icon.png", + new TextReader( + '' + ) + ); + const artifact = await built(manifest(), await writer.close()); + const response = await get( + `${root}/packages/${artifact.id}/icon`, + await authHeaders("owner") + ); + expect(response.headers.get("Content-Type")).toBe("image/svg+xml"); + expect(await response.text()).not.toContain("onload"); + }); + it.each(["png", "webp", "avif"] as const)( + "converts icon.%s to bounded AVIF and PNG images", + async (format) => { + const svg = + ''; + const image = await env + .PACKAGE_IMAGES!.input(new Blob([svg]).stream()) + .output({ format: `image/${format}` }); + const original = new Uint8Array(await image.response().arrayBuffer()); + const writer = new ZipWriter(new Uint8ArrayWriter(), options); + await writer.add(`icon.${format}`, new Uint8ArrayReader(original)); + const artifact = await built(manifest(), await writer.close()); + const response = await get( + `${root}/packages/${artifact.id}/icon`, + await authHeaders("owner") + ); + const avif = await get(`${root}/packages/${artifact.id}/icon`, { + ...(await authHeaders("owner")), + Accept: "image/avif,image/png" + }); + expect(avif.headers.get("Content-Type")).toBe("image/avif"); + expect( + new TextDecoder().decode((await avif.arrayBuffer()).slice(4, 12)) + ).toBe("ftypavif"); + expect( + ( + await get(`${root}/packages/${artifact.id}/icon`, { + ...(await authHeaders("owner")), + Accept: "image/avif;q=0,image/png" + }) + ).headers.get("Content-Type") + ).toBe("image/png"); + const info = await env.PACKAGE_IMAGES!.info(response.body!); + expect(info).toMatchObject({ + format: "image/png", + width: 256, + height: 128 + }); + const download = await get( + `${root}/packages/${artifact.id}/download`, + await authHeaders("owner") + ); + const zip = new ZipReader( + new Uint8ArrayReader(new Uint8Array(await download.arrayBuffer())), + options + ); + try { + const entry = (await zip.getEntries()).find( + (entry) => entry.filename === `icon.${format}` + )!; + if (entry.directory) throw new Error("Expected icon file"); + expect(await entry.getData(new Uint8ArrayWriter())).toEqual(original); + } finally { + await zip.close(); + } + } + ); + + it("lets moderators rebuild published source for a correction without uploading new files", async () => { + const artifact = await built(); + const revision = await submitted(artifact); + expect((await approve(revision)).status).toBe(200); + const corrected = { ...manifest(), description: "Moderator correction" }; + const response = await upload( + corrected, + undefined, + artifact.source_id, + "moderator" + ); + expect(response.status).toBe(201); + const next = PackageArtifactSchema.parse( + ((await response.json()) as { result: unknown }).result + ); + expect(next.source_id).toBe(artifact.source_id); + expect(next.manifest.description).toBe(corrected.description); + expect( + (await upload(corrected, await zip(), undefined, "moderator")).status + ).toBe(403); + expect( + ( + await post( + `${root}/extensions/package-demo/moderator-correct`, + await authHeaders("moderator"), + { + ...content(next), + correction_note: "Correct the package metadata" + } + ) + ).status + ).toBe(200); + }); + + it("rejects invalid icons without reserving the source version", async () => { + const writer = new ZipWriter(new Uint8ArrayWriter(), options); + await writer.add("icon.png", new TextReader("not an image")); + const response = await upload(manifest(), await writer.close()); + expect(response.status).toBe(422); + expect(await response.json()).toMatchObject({ + error: { code: "INVALID_ICON" } + }); + expect( + await db.prepare("SELECT count(*) AS count FROM package_sources").first() + ).toEqual({ count: 0 }); + expect((await upload()).status).toBe(201); + }); + + function configureB2() { + Object.assign(env, { + EXTENSION_ASSETS_S3_ENDPOINT: "https://s3.eu-central-003.backblazeb2.com", + EXTENSION_ASSETS_S3_REGION: "eu-central-003", + EXTENSION_ASSETS_S3_BUCKET: "test-assets", + EXTENSION_ASSETS_S3_ACCESS_KEY_ID: "test-key", + EXTENSION_ASSETS_S3_SECRET_ACCESS_KEY: "test-secret", + EXTENSION_ASSETS_PUBLIC_URL: "https://extensions-assets.fossbilling.net", + EXTENSION_ASSETS_S3_PREFIX: `tests-${crypto.randomUUID()}` + }); + const calls: Request[] = []; + let failure = false; + vi.stubGlobal( + "fetch", + async (input: RequestInfo | URL, init?: RequestInit) => { + const request = + input instanceof Request ? input : new Request(input, init); + expect(new URL(request.url).hostname).toBe( + "s3.eu-central-003.backblazeb2.com" + ); + calls.push(request.clone()); + return new Response(null, { + status: failure ? 403 : request.method === "HEAD" ? 404 : 200 + }); + } + ); + return { + calls, + fail: (value: boolean) => { + failure = value; + } + }; + } + async function iconSource(format: "svg" | "png" = "svg") { + const svg = + ''; + const bytes = + format === "svg" + ? new TextEncoder().encode(svg) + : new Uint8Array( + await ( + await env + .PACKAGE_IMAGES!.input(new Blob([svg]).stream()) + .output({ format: "image/png" }) + ) + .response() + .arrayBuffer() + ); + const writer = new ZipWriter(new Uint8ArrayWriter(), options); + await writer.add( + `icon.${format}`, + new Uint8ArrayReader(new Uint8Array(bytes)) + ); + return writer.close(); + } + it("keeps pending images private and publishes approved images to B2 only once", async () => { + const remote = configureB2(); + const artifact = await built(manifest(), await iconSource()); + const pending = await submitted(artifact); + expect( + ( + await get( + `${root}/packages/${artifact.id}/icon`, + await authHeaders("owner") + ) + ).status + ).toBe(200); + expect(remote.calls).toHaveLength(0); + expect((await approve(pending)).status).toBe(200); + const published = await getExtension(db, "package-demo"); + expect(published?.icon_url).toMatch( + /^https:\/\/extensions-assets\.fossbilling\.net\/tests-[a-f0-9-]+\/icons\/[a-f0-9]{64}\.svg$/ + ); + expect(remote.calls.map((request) => request.method)).toEqual([ + "HEAD", + "PUT" + ]); + const upload = remote.calls[1]; + expect(upload.headers.get("Authorization")).toContain( + "AWS4-HMAC-SHA256 Credential=test-key/" + ); + expect(upload.headers.get("Cache-Control")).toBe( + "public, max-age=31536000, immutable" + ); + expect(upload.headers.get("Content-Type")).toBe("image/svg+xml"); + expect(await sha256(new Uint8Array(await upload.arrayBuffer()))).toBe( + upload.headers.get("x-amz-meta-sha256") + ); + await get(`${root}/packages/${artifact.id}/icon`, {}); + expect(remote.calls).toHaveLength(2); + }); + it("keeps approval and fallback delivery working during B2 failures, then retries publication", async () => { + const remote = configureB2(); + remote.fail(true); + const artifact = await built(manifest(), await iconSource()); + expect((await approve(await submitted(artifact))).status).toBe(200); + expect((await getExtension(db, "package-demo"))?.icon_url).toBe( + artifact.download_url.replace(/\/download$/, "/icon") + ); + expect((await get(`${root}/packages/${artifact.id}/icon`, {})).status).toBe( + 200 + ); + expect(remote.calls.every((request) => request.method === "HEAD")).toBe( + true + ); + remote.fail(false); + expect((await get(`${root}/packages/${artifact.id}/icon`, {})).status).toBe( + 200 + ); + expect((await getExtension(db, "package-demo"))?.icon_url).toContain( + "https://extensions-assets.fossbilling.net/tests-" + ); + }); + it("publishes AVIF and PNG with one immutable image identity for browser fallback", async () => { + const remote = configureB2(); + const artifact = await built(manifest(), await iconSource("png")); + expect((await approve(await submitted(artifact))).status).toBe(200); + const uploads = remote.calls.filter((request) => request.method === "PUT"); + expect(uploads).toHaveLength(2); + expect(uploads[0].url.replace(/\.avif$/, ".png")).toBe(uploads[1].url); + expect( + uploads.map((request) => request.headers.get("Content-Type")) + ).toEqual(["image/avif", "image/png"]); + expect((await getExtension(db, "package-demo"))?.icon_url).toBe( + uploads[1].url.replace( + "https://s3.eu-central-003.backblazeb2.com/test-assets", + "https://extensions-assets.fossbilling.net" + ) + ); + }); + + it("serves a generic icon for an authorized package without an icon", async () => { + const artifact = await built(); + const response = await get( + `${root}/packages/${artifact.id}/icon`, + await authHeaders("owner") + ); + expect(response.status).toBe(200); + expect(await response.text()).toContain(" { + const source = await zip(); + const first = await built(manifest(), source); + const retry = await built(manifest(), source); + expect(retry.id).toBe(first.id); + const second = await built( + { ...manifest(), description: "Corrected metadata" }, + undefined, + first.source_id + ); + expect(second.source_id).toBe(first.source_id); + expect(second.manifest.version).toBe(first.manifest.version); + expect(second.id).not.toBe(first.id); + expect(second.sha256).not.toBe(first.sha256); + expect(second.download_url).not.toBe(first.download_url); + const rows = await db + .prepare("SELECT count(*) AS count FROM package_objects") + .first<{ count: number }>(); + expect(rows!.count).toBe(3); + }); + + it("reuses a ready artifact without reopening its stored source", async () => { + const artifact = await built(); + const source = await db + .prepare("SELECT object_id FROM package_sources WHERE id=?") + .bind(artifact.source_id) + .first<{ object_id: string }>(); + await bucket().delete(source!.object_id); + expect((await built(manifest(), undefined, artifact.source_id)).id).toBe( + artifact.id + ); + expect( + ( + await upload( + { ...manifest(), description: "Needs a rebuild" }, + undefined, + artifact.source_id + ) + ).status + ).toBe(503); + }); + + it("requires a new version for changed source and hides foreign stored sources", async () => { + const first = await built(); + expect((await upload(manifest(), await zip("changed source"))).status).toBe( + 409 + ); + const changed = { ...manifest(), version: "1.1.0" }; + expect((await upload(changed, await zip("changed source"))).status).toBe( + 201 + ); + await seedDeveloper("other-developer", "other"); + expect( + (await upload(manifest(), undefined, first.source_id, "other")).status + ).toBe(404); + }); + + it("rejects metadata/requirements/URL/checksum mismatches at submission", async () => { + const artifact = await built(); + for (const patch of [ + { description: "Not in ZIP" }, + { download_url: "https://evil.test/not-the-package.zip" }, + { requires: { fossbilling: ">=9.0.0" } }, + { + releases: [{ ...content(artifact).releases[0], sha256: "0".repeat(64) }] + }, + { + releases: [ + { + ...content(artifact).releases[0], + requires: { fossbilling: ">=9.0.0" } + } + ] + } + ]) { + const response = await post( + `${root}/extensions`, + await authHeaders("owner"), + { id: "package-demo", ...content(artifact), ...patch } + ); + expect(response.status).toBe(409); + } + expect( + await db + .prepare( + "SELECT count(*) AS count FROM extensions WHERE id='package-demo'" + ) + .first() + ).toEqual({ count: 0 }); + }); + + it("replaces a pending package, invalidates the old review, and preserves previously published bytes", async () => { + const first = await built(); + const revision = await submitted(first); + expect((await approve(revision)).status).toBe(200); + const second = await built( + { ...manifest(), description: "Correction" }, + undefined, + first.source_id + ); + const proposed = await put( + `${root}/extensions/package-demo`, + await authHeaders("owner"), + content(second) + ); + expect(proposed.status).toBe(202); + const oldPending = ( + (await proposed.json()) as { result: { revision_id: string } } + ).result.revision_id; + const third = await built( + { ...manifest(), description: "Final correction" }, + undefined, + first.source_id + ); + const replaced = await put( + `${root}/extensions/package-demo/revisions/${oldPending}`, + await authHeaders("owner"), + content(third) + ); + expect(replaced.status).toBe(202); + expect((await approve(oldPending)).status).toBe(404); + expect( + (await get(`${root}/packages/${second.id}/download`, {})).status + ).toBe(404); + expect( + ( + await approve( + ((await replaced.json()) as { result: { revision_id: string } }) + .result.revision_id + ) + ).status + ).toBe(200); + expect( + (await get(`${root}/packages/${first.id}/download`, {})).status + ).toBe(200); + expect( + (await get(`${root}/packages/${third.id}/download`, {})).status + ).toBe(200); + }); + + it("prevents reverting a managed listing to external downloads or mutating artifacts", async () => { + const artifact = await built(); + expect((await approve(await submitted(artifact))).status).toBe(200); + expect( + ( + await put( + `${root}/extensions/package-demo`, + await authHeaders("owner"), + sampleContent() + ) + ).status + ).toBe(409); + await expect( + db + .prepare("UPDATE package_artifacts SET sha256=? WHERE id=?") + .bind("0".repeat(64), artifact.id) + .run() + ).rejects.toThrow("immutable"); + await expect( + db + .prepare("UPDATE package_sources SET sha256=? WHERE id=?") + .bind("0".repeat(64), artifact.source_id) + .run() + ).rejects.toThrow("immutable"); + }); + + it("revalidates the artifact at approval rather than trusting submission-time checks", async () => { + const artifact = await built(); + const revision = await submitted(artifact); + await db + .prepare( + "UPDATE extension_revisions SET content=json_set(content,'$.description','Changed after submission') WHERE id=?" + ) + .bind(revision) + .run(); + expect((await approve(revision)).status).toBe(409); + expect( + (await get(`${root}/packages/${artifact.id}/download`, {})).status + ).toBe(404); + }); + + it("detects corrupted stored source and package objects", async () => { + const artifact = await built(); + const source = await db + .prepare("SELECT object_id FROM package_sources WHERE id=?") + .bind(artifact.source_id) + .first<{ object_id: string }>(); + await bucket().put(source!.object_id, "corrupt"); + expect( + ( + await upload( + { ...manifest(), description: "Correction" }, + undefined, + artifact.source_id + ) + ).status + ).toBe(503); + const stored = await db + .prepare("SELECT object_id FROM package_artifacts WHERE id=?") + .bind(artifact.id) + .first<{ object_id: string }>(); + await bucket().put(stored!.object_id, "corrupt"); + expect( + ( + await get( + `${root}/packages/${artifact.id}/download`, + await authHeaders("owner") + ) + ).status + ).toBe(503); + }); + + it("concurrent identical uploads converge on one source and artifact", async () => { + const source = await zip(); + const [first, second] = await Promise.all([ + built(manifest(), source), + built(manifest(), source) + ]); + expect(first.id).toBe(second.id); + expect( + await db.prepare("SELECT count(*) AS count FROM package_sources").first() + ).toEqual({ count: 1 }); + expect( + await db + .prepare("SELECT count(*) AS count FROM package_artifacts") + .first() + ).toEqual({ count: 1 }); + expect( + await db.prepare("SELECT count(*) AS count FROM package_objects").first() + ).toEqual({ count: 3 }); // one duplicate staged upload remains eligible for cleanup + }); + + it("does not allow a removed managed release to return as an external download", async () => { + const first = await built(); + expect((await approve(await submitted(first))).status).toBe(200); + const second = await built({ ...manifest(), version: "1.1.0" }); + const nextContent = content(second); + const proposal = await put( + `${root}/extensions/package-demo`, + await authHeaders("owner"), + nextContent + ); + expect(proposal.status).toBe(202); + expect( + ( + await approve( + ((await proposal.json()) as { result: { revision_id: string } }) + .result.revision_id + ) + ).status + ).toBe(200); + const response = await put( + `${root}/extensions/package-demo`, + await authHeaders("owner"), + { + ...nextContent, + releases: [...nextContent.releases, sampleContent().releases[0]] + } + ); + expect(response.status).toBe(409); + }); + + it("publishes a manually selected older release and rejects competing latest flags", async () => { + const first = await built(); + const newer = await built({ ...manifest(), version: "2.0.0" }); + const payload = { + ...content(first), + releases: [ + { ...content(first).releases[0], is_latest: true }, + ...content(newer).releases + ] + }; + const conflict = await post( + `${root}/extensions`, + await authHeaders("owner"), + { + id: "package-demo", + ...payload, + releases: payload.releases.map((release) => ({ + ...release, + is_latest: true + })) + } + ); + expect(conflict.status).toBe(409); + const response = await post( + `${root}/extensions`, + await authHeaders("owner"), + { id: "package-demo", ...payload } + ); + expect(response.status).toBe(201); + expect( + ( + await approve( + ((await response.json()) as { result: { revision_id: string } }) + .result.revision_id + ) + ).status + ).toBe(200); + const version = await get("/extensions/v1/package-demo/version", {}); + expect(await version.text()).toBe("1.0.0"); + expect((await getExtension(db, "package-demo"))!.version).toBe("1.0.0"); + }); + + it("keeps production disabled until explicitly enabled with complete storage", async () => { + env.ENVIRONMENT = "production"; + expect((await upload()).status).toBe(503); // local mode cannot enable production + Object.assign(env, { EXTENSION_PACKAGES_MODE: "enabled" }); + expect((await upload()).status).toBe(503); // localhost URL / incomplete storage + env.EXTENSION_PACKAGES_BASE_URL = + "https://api.fossbilling.net/extensions/v2"; + expect((await upload()).status).toBe(503); // must not fall back to local R2 + Object.assign(env, { + EXTENSION_PACKAGES_S3_ENDPOINT: "https://s3.example.test", + EXTENSION_PACKAGES_S3_REGION: "eu-central-003", + EXTENSION_PACKAGES_S3_BUCKET: "private-packages", + EXTENSION_PACKAGES_S3_PREFIX: "production", + EXTENSION_PACKAGES_S3_ACCESS_KEY_ID: "test-key", + EXTENSION_PACKAGES_S3_SECRET_ACCESS_KEY: "test-secret" + }); + const requests: Request[] = []; + vi.stubGlobal("fetch", async (request: Request) => { + requests.push(request); + return new Response(null, { status: 200 }); + }); + const response = await upload( + manifest(), + undefined, + undefined, + "owner", + false + ); + expect(response.status).toBe(202); + expect(response.headers.get("Location")).toMatch( + /^https:\/\/api\.fossbilling\.net\/extensions\/v2\/package-jobs\// + ); + expect(requests.length).toBeGreaterThan(0); + expect( + requests.every((request) => request.url.includes("/production/packages/")) + ).toBe(true); + Object.assign(env, { EXTENSION_PACKAGES_S3_PREFIX: "local" }); + expect((await upload()).status).toBe(503); + delete env.EXTENSION_PACKAGES_S3_PREFIX; + env.ENVIRONMENT = "local"; + }); + + it("checks account ownership, upload bounds, invalid archives and disabled storage", async () => { + expect((await upload(manifest(), new Uint8Array([1, 2, 3]))).status).toBe( + 422 + ); + expect( + (await upload(manifest(), new Uint8Array(5 * 1024 * 1024 + 1))).status + ).toBe(413); + await db + .prepare("UPDATE users SET deleted_at=CURRENT_TIMESTAMP WHERE id='owner'") + .run(); + expect((await upload()).status).toBe(403); + env.EXTENSION_PACKAGES_MODE = "disabled"; + expect((await upload()).status).toBe(503); + }); + + it("leaves failed uploads reserved for cleanup and permits a safe retry", async () => { + const failure = vi + .spyOn(bucket(), "put") + .mockRejectedValueOnce(new Error("storage outage")); + try { + expect((await upload()).status).toBe(503); + } finally { + failure.mockRestore(); + } + expect( + await db + .prepare("SELECT count(*) AS count FROM package_artifacts") + .first() + ).toEqual({ count: 0 }); + expect( + await db.prepare("SELECT count(*) AS count FROM package_objects").first() + ).toEqual({ count: 1 }); + expect((await upload()).status).toBe(201); + expect( + await db.prepare("SELECT count(*) AS count FROM package_sources").first() + ).toEqual({ count: 1 }); + }); + + it("counts abandoned reservations against quota and safely cleans them after a day", async () => { + const artifact = await built(); + await db + .prepare( + "INSERT INTO package_objects (id,created_by,bytes,created_at) VALUES ('abandoned','owner',?,datetime('now','-2 days'))" + ) + .bind(512 * 1024 * 1024) + .run(); + await bucket().put("abandoned", "failed upload"); + expect( + ( + await upload( + { ...manifest(), description: "quota test" }, + undefined, + artifact.source_id + ) + ).status + ).toBe(409); + await insertUser(db, { id: "moderator", is_moderator: 1 }); + const cleanup = await post( + `${root}/packages/cleanup`, + await authHeaders("moderator") + ); + expect(cleanup.status).toBe(200); + expect(await cleanup.json()).toEqual({ result: { deleted: 1 } }); + expect(await bucket().get("abandoned")).toBeNull(); + expect( + ( + await upload( + { ...manifest(), description: "quota test" }, + undefined, + artifact.source_id + ) + ).status + ).toBe(201); + }); +}); diff --git a/test/services/extensions/v2/resource-bounds.test.ts b/test/services/extensions/v2/resource-bounds.test.ts index 6d031ced..6f7bc102 100644 --- a/test/services/extensions/v2/resource-bounds.test.ts +++ b/test/services/extensions/v2/resource-bounds.test.ts @@ -188,7 +188,6 @@ describe("Extension resource admission", () => { "releases", "website", "license", - "icon_url", "readme", "source", "version", diff --git a/vitest.config.ts b/vitest.config.ts index 5555db35..6e9e4b8e 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -18,6 +18,8 @@ export default defineConfig({ cloudflareTest({ wrangler: { configPath: "./wrangler.jsonc" }, miniflare: { + r2Buckets: ["EXTENSION_PACKAGE_BUCKET", "EXTENSION_IMAGE_BUCKET"], + images: { binding: "PACKAGE_IMAGES" }, // Add test environment variables bindings: { GITHUB_TOKEN: "test-github-token", @@ -68,7 +70,8 @@ export default defineConfig({ exclude: [ "**/node_modules/**", "**/test/lib/adapters/node/**", - "**/test/services/extensions/v2/migrations.test.ts" + "**/test/services/extensions/v2/migrations.test.ts", + "**/*.node.test.ts" ], // Test timeout configuration diff --git a/vitest.node.config.ts b/vitest.node.config.ts index d127eb85..2a7b7a6c 100644 --- a/vitest.node.config.ts +++ b/vitest.node.config.ts @@ -5,7 +5,8 @@ export default defineConfig({ environment: "node", include: [ "test/lib/adapters/node/**/*.test.ts", - "test/services/extensions/v2/migrations.test.ts" + "test/services/extensions/v2/migrations.test.ts", + "test/services/extensions/v2/package-jobs.node.test.ts" ], testTimeout: 10000, diff --git a/worker-configuration.d.ts b/worker-configuration.d.ts index 45537ba4..bc5958a3 100644 --- a/worker-configuration.d.ts +++ b/worker-configuration.d.ts @@ -1,34 +1,78 @@ /* eslint-disable */ -// Generated by Wrangler by running `wrangler types --env-interface=CloudflareBindings` (hash: e27e965b107adae819c46ccc0379af8f) -// Runtime types generated with workerd@1.20261006.1 2026-10-01 +// Generated by Wrangler by running `wrangler types --env-interface=CloudflareBindings` (hash: 26fdf3e2c27ec137e614462c60dbc01c) +// Runtime types generated with workerd@1.20261103.1 2026-10-01 interface __BaseEnv_CloudflareBindings { AUTH_KV: KVNamespace; CACHE_KV: KVNamespace; DOWNLOAD_BUCKET: R2Bucket; + EXTENSION_PACKAGE_BUCKET?: R2Bucket; + EXTENSION_IMAGE_BUCKET: R2Bucket; DB_CENTRAL_ALERTS: D1Database; DB_EXTENSIONS: D1Database; + EXTENSION_PACKAGE_QUEUE: Queue; EXTENSION_WRITE_RATE_LIMITER: RateLimit; PROFILE_CREATION_RATE_LIMITER: RateLimit; - ENVIRONMENT: "production"; + PACKAGE_IMAGES: ImagesBinding; + ENVIRONMENT: "local" | "production"; EXTENSIONS_RETENTION_MODE: "dry-run"; - EXTENSIONS_V2_EMAIL_PROVIDER: "mxroute"; - EXTENSIONS_V2_MXROUTE_SERVER: "wednesday.mxrouting.net"; - EXTENSIONS_V2_MXROUTE_USERNAME: "extensions@fossbilling.org"; + M2M_ISSUER: "https://auth.fossbilling.net"; + M2M_JWKS_URL: "https://auth.fossbilling.net/.well-known/jwks.json"; + M2M_AUDIENCE: "https://api.fossbilling.net"; + EXTENSIONS_V2_EMAIL_PROVIDER: "disabled" | "mxroute"; + EXTENSIONS_V2_MXROUTE_SERVER: "" | "wednesday.mxrouting.net"; + EXTENSIONS_V2_MXROUTE_USERNAME: "" | "extensions@fossbilling.org"; + EXTENSION_PACKAGES_MODE: "local" | "disabled"; + EXTENSION_PACKAGES_BASE_URL: "http://localhost:8789/extensions/v2" | "https://api.fossbilling.net/extensions/v2"; GITHUB_TOKEN: string; ASSERTION_SIGNING_SECRET: string; EXTENSIONS_V2_MXROUTE_PASSWORD: string; EXTENSIONS_REVALIDATE_SECRET: string; - M2M_ISSUER: "https://auth.fossbilling.net"; - M2M_JWKS_URL: "https://auth.fossbilling.net/.well-known/jwks.json"; - M2M_AUDIENCE: "https://api.fossbilling.net"; PREVIEW_GITHUB_BUDGET: DurableObjectNamespace; EXTENSIONS_FRONTEND: Fetcher /* extensions */; + EXTENSION_PACKAGES_S3_ENDPOINT?: "https://s3.eu-central-003.backblazeb2.com"; + EXTENSION_PACKAGES_S3_REGION?: "eu-central-003"; + EXTENSION_PACKAGES_S3_BUCKET?: "fb-extensions-packages"; + EXTENSION_PACKAGES_S3_PREFIX?: "production"; + EXTENSION_ASSETS_S3_ENDPOINT?: "https://s3.eu-central-003.backblazeb2.com"; + EXTENSION_ASSETS_S3_REGION?: "eu-central-003"; + EXTENSION_ASSETS_S3_BUCKET?: "fb-extensions-assets"; + EXTENSION_ASSETS_S3_PREFIX?: "production"; + EXTENSION_ASSETS_PUBLIC_URL?: "https://extensions-assets.fossbilling.net"; } declare namespace Cloudflare { interface GlobalProps { mainModule: typeof import("./src/app/index"); durableNamespaces: "PreviewGitHubBudget"; } + interface PackagesLocalEnv { + AUTH_KV: KVNamespace; + CACHE_KV: KVNamespace; + DOWNLOAD_BUCKET: R2Bucket; + EXTENSION_PACKAGE_BUCKET: R2Bucket; + EXTENSION_IMAGE_BUCKET: R2Bucket; + DB_CENTRAL_ALERTS: D1Database; + DB_EXTENSIONS: D1Database; + EXTENSION_PACKAGE_QUEUE: Queue; + EXTENSION_WRITE_RATE_LIMITER: RateLimit; + PROFILE_CREATION_RATE_LIMITER: RateLimit; + PACKAGE_IMAGES: ImagesBinding; + ENVIRONMENT: "local"; + EXTENSIONS_RETENTION_MODE: "dry-run"; + M2M_ISSUER: "https://auth.fossbilling.net"; + M2M_JWKS_URL: "https://auth.fossbilling.net/.well-known/jwks.json"; + M2M_AUDIENCE: "https://api.fossbilling.net"; + EXTENSIONS_V2_EMAIL_PROVIDER: "disabled"; + EXTENSIONS_V2_MXROUTE_SERVER: ""; + EXTENSIONS_V2_MXROUTE_USERNAME: ""; + EXTENSION_PACKAGES_MODE: "local"; + EXTENSION_PACKAGES_BASE_URL: "http://localhost:8789/extensions/v2"; + GITHUB_TOKEN: string; + ASSERTION_SIGNING_SECRET: string; + EXTENSIONS_V2_MXROUTE_PASSWORD: string; + EXTENSIONS_REVALIDATE_SECRET: string; + PREVIEW_GITHUB_BUDGET: DurableObjectNamespace; + EXTENSIONS_FRONTEND: Fetcher /* extensions */; + } interface Env extends __BaseEnv_CloudflareBindings {} } interface CloudflareBindings extends __BaseEnv_CloudflareBindings {} @@ -36,7 +80,7 @@ type StringifyValues> = { [Binding in keyof EnvType]: EnvType[Binding] extends string ? EnvType[Binding] : string; }; declare namespace NodeJS { - interface ProcessEnv extends StringifyValues> {} + interface ProcessEnv extends StringifyValues> {} } declare module "*.txt" { const value: string; @@ -488,6 +532,7 @@ interface ExecutionContext { readonly props: Props; cache?: CacheContext; readonly access?: CloudflareAccessContext; + mapVirtualHost(fetcher: Fetcher, port: number): string; tracing: Tracing; abort(reason?: any): void; } @@ -3346,13 +3391,7 @@ declare abstract class SqlStorageStatement { } type SqlStorageValue = ArrayBuffer | string | number | null; declare abstract class SqlStorageCursor> { - next(): { - done?: false; - value: T; - } | { - done: true; - value?: never; - }; + next(): IteratorResult; toArray(): T[]; one(): T; raw(): IterableIterator; @@ -3511,9 +3550,9 @@ interface ContainerSnapshotOptions { } type ContainerStartupOptions = { entrypoint?: string[]; - enableInternet: boolean; + enableInternet?: boolean; env?: Record; - instance?: "lite" | "standard-1" | "standard-2" | "standard-3" | "standard-4" | ContainerStartResources; + instance?: "lite" | "basic" | "standard-1" | "standard-2" | "standard-3" | "standard-4" | ContainerStartResources; labels?: Record; directorySnapshots?: ContainerDirectorySnapshotRestoreParams[]; } & ({ @@ -3678,6 +3717,7 @@ interface Tracing { } declare abstract class Span { get isTraced(): boolean; + spanContext(): TracingSpanContext; setAttribute(key: string, value: boolean | number | string): this; setAttributes(attributes: Record): this; recordException(exception: string | { @@ -3705,6 +3745,11 @@ interface TracingSpanStatus { code: TracingSpanStatusCode; message?: string; } +interface TracingSpanContext { + traceId: string; + spanId: string; + traceFlags: number; +} /** * Represents the identity of a user authenticated via Cloudflare Access. * This matches the result of calling /cdn-cgi/access/get-identity. @@ -11583,20 +11628,16 @@ interface BrowserRunBaseOptions { */ cacheTTL?: number; } -/** - * Backend selection, mixed into the options of the quick actions that support it. - * Deliberately not part of `BrowserRunBaseOptions`: `scrape`, `links` and `snapshot` - * reject an alternate backend, so they must not accept the field. - */ +/** Backend selection for quick actions. */ interface BrowserRunAlternateBackendOptions { /** Render with an alternate browser backend instead of the default one. */ browser?: 'kitesurf'; } /** Common options shared by all quick actions. Exactly one of `url` or `html` must be provided.*/ -type BrowserRunCommonOptions = (BrowserRunBaseOptions & { +type BrowserRunCommonOptions = BrowserRunBaseOptions & BrowserRunAlternateBackendOptions & ({ /** URL to navigate to, e.g. `"https://example.com"`. */ url: string; -}) | (BrowserRunBaseOptions & { +} | { /** Set the HTML content of the page directly. */ html: string; }); @@ -11626,7 +11667,7 @@ type BrowserRunScreenshotOptions = BrowserRunCommonOptions & { scrollPage?: boolean; /** @see https://pptr.dev/api/puppeteer.screenshotoptions */ screenshotOptions?: BrowserRunPuppeteerScreenshotOptions; -} & BrowserRunAlternateBackendOptions; +}; type BrowserRunPDFOptions = BrowserRunCommonOptions & { /** @see https://pptr.dev/api/puppeteer.pdfoptions */ pdfOptions?: { @@ -11662,7 +11703,7 @@ type BrowserRunPDFOptions = BrowserRunCommonOptions & { /** @default 30000 */ timeout?: number; }; -} & BrowserRunAlternateBackendOptions; +}; type BrowserRunScrapeOptions = BrowserRunCommonOptions & { /** CSS selectors to scrape. At least one element is required. */ elements: Array<{ @@ -11697,7 +11738,7 @@ type BrowserRunAccessibilityTreeOptions = BrowserRunCommonOptions & { * HTTP 200; a malformed selector is an error. */ root?: string; -} & BrowserRunAlternateBackendOptions; +}; interface BrowserRunJsonBaseOptions { /** Custom AI models to try in order. Max 3. Falls back to next on error. */ custom_ai?: Array<{ @@ -11711,7 +11752,7 @@ interface BrowserRunJsonBaseOptions { * Options for the `json` quick action. * At least one of `prompt` or `response_format` must be provided. */ -type BrowserRunJsonOptions = BrowserRunCommonOptions & BrowserRunAlternateBackendOptions & BrowserRunJsonBaseOptions & ({ +type BrowserRunJsonOptions = BrowserRunCommonOptions & BrowserRunJsonBaseOptions & ({ /** Natural-language prompt describing what data to extract. */ prompt: string; /** Structured output schema for the AI model. @see https://developers.cloudflare.com/workers-ai/json-mode/ */ @@ -11722,8 +11763,8 @@ type BrowserRunJsonOptions = BrowserRunCommonOptions & BrowserRunAlternateBacken /** Structured output schema for the AI model. @see https://developers.cloudflare.com/workers-ai/json-mode/ */ response_format: AiTextGenerationResponseFormat; }); -type BrowserRunContentOptions = BrowserRunCommonOptions & BrowserRunAlternateBackendOptions; -type BrowserRunMarkdownOptions = BrowserRunCommonOptions & BrowserRunAlternateBackendOptions; +type BrowserRunContentOptions = BrowserRunCommonOptions; +type BrowserRunMarkdownOptions = BrowserRunCommonOptions; type BrowserRunRedirectHop = { /** URL that returned the redirect. */ url: string; @@ -13261,20 +13302,30 @@ interface Disposable { } declare module 'cloudflare:durable-objects' { /** - * Marks a Durable Object method as safe to run more than once. The runtime may then retry calls - * to it after a disconnect that might have happened after the method started. It does not + * Marks Durable Object methods as safe to run more than once. The runtime may then retry calls + * to them after a disconnect that might have happened after the method started. It does not * enable retries or change how many are made. * + * Mark methods on the class prototype, for example from a static block: + * + * ```ts + * class Counter extends DurableObject { + * static { + * retryable(this.prototype.reset); + * } + * async reset() {} + * } + * ``` + * + * The mark is on the function itself. A subclass inherits it, but an override, a bound copy, or + * a wrapper is not marked unless it is also passed to `retryable()`. + * * Each attempt creates its own return value, so if the method returns an `RpcTarget`, its * `[Symbol.dispose]()` may run once per attempt. * - * Only public instance methods can be decorated. When composing decorators, apply `@retryable` - * outermost (first in source order) so it marks the method that is finally installed. - * Otherwise `@retryable` has no effect, and constructing the object logs a warning. - * Requires standard (not `experimentalDecorators`) decorators and a bundler that transforms - * them, such as Wrangler. + * @throws TypeError if an argument is not a function. */ - export function retryable(value: (this: This, ...args: Args) => Return, context: ClassMethodDecoratorContext Return>): (this: This, ...args: Args) => Return; + export function retryable(...methods: ((...args: never[]) => unknown)[]): void; } /** * The returned data after sending an email @@ -13416,7 +13467,10 @@ declare module "cloudflare:email" { * Evaluation context for targeting rules. * Keys are attribute names (e.g. "userId", "country"), values are the attribute values. */ -type FlagshipEvaluationContext = Record; +type FlagshipEvaluationContextValue = string | number | boolean | null | FlagshipEvaluationContextValue[] | { + [key: string]: FlagshipEvaluationContextValue; +}; +type FlagshipEvaluationContext = Record; interface FlagshipEvaluationDetails { flagKey: string; value: T; @@ -13425,6 +13479,7 @@ interface FlagshipEvaluationDetails { errorCode?: string | undefined; errorMessage?: string | undefined; } +type FlagshipWidenedValue = T extends boolean ? boolean : T extends string ? string : T extends number ? number : T; interface FlagshipEvaluationError extends Error { } /** @@ -13452,8 +13507,23 @@ declare abstract class Flagship { * @param flagKey The key of the flag to evaluate. * @param defaultValue Optional default value returned when evaluation fails. * @param context Optional evaluation context for targeting rules. + * @deprecated Use getValue() instead. */ get(flagKey: string, defaultValue?: unknown, context?: FlagshipEvaluationContext): Promise; + /** + * Get a flag value, inferring its expected type from the default value. + * @param flagKey The key of the flag to evaluate. + * @param defaultValue Default value returned when evaluation fails or the flag type does not match. + * @param context Optional evaluation context for targeting rules. + */ + getValue | unknown[]>(flagKey: string, defaultValue: T, context?: FlagshipEvaluationContext): Promise>; + /** + * Get a flag value with full evaluation details, inferring its expected type from the default value. + * @param flagKey The key of the flag to evaluate. + * @param defaultValue Default value returned when evaluation fails or the flag type does not match. + * @param context Optional evaluation context for targeting rules. + */ + getDetails | unknown[]>(flagKey: string, defaultValue: T, context?: FlagshipEvaluationContext): Promise>>; /** * Get a boolean flag value. * @param flagKey The key of the flag to evaluate. @@ -14254,7 +14324,7 @@ declare namespace Rpc { // Structured cloneables BaseType // Structured cloneable composites - | Map ? Serializable : never, T extends Map ? Serializable : never> | Set ? Serializable : never> | ReadonlyArray ? Serializable : never> | { + | Map ? Serializable : never, T extends Map ? Serializable : never> | Set ? Serializable : never> | ReadonlyMap ? Serializable : never, T extends ReadonlyMap ? Serializable : never> | ReadonlySet ? Serializable : never> | ReadonlyArray ? Serializable : never> | { [K in keyof T]: K extends number | string ? Serializable : never; } // Special types @@ -14272,7 +14342,7 @@ declare namespace Rpc { type BaseType = void | undefined | null | boolean | number | bigint | string | TypedArray | ArrayBuffer | DataView | Date | Error | RegExp | ReadableStream | WritableStream | Request | Response | Headers; // Recursively rewrite all `Stubable` types with `Stub`s // prettier-ignore - type Stubify = T extends Stubable ? Stub : T extends Map ? Map, Stubify> : T extends Set ? Set> : T extends Array ? Array> : T extends ReadonlyArray ? ReadonlyArray> : T extends BaseType ? T : T extends { + type Stubify = T extends Stubable ? Stub : T extends Map ? Map, Stubify> : T extends Set ? Set> : T extends ReadonlyMap ? ReadonlyMap, Stubify> : T extends ReadonlySet ? ReadonlySet> : T extends Array ? Array> : T extends ReadonlyArray ? ReadonlyArray> : T extends BaseType ? T : T extends { [key: string | number]: any; } ? { [K in keyof T]: Stubify; @@ -14281,7 +14351,7 @@ declare namespace Rpc { // Note we use `StubBase` instead of `Stub` here to avoid circular dependencies: // `Stub` depends on `Provider`, which depends on `Unstubify`, which would depend on `Stub`. // prettier-ignore - type Unstubify = T extends StubBase ? V : T extends Map ? Map, Unstubify> : T extends Set ? Set> : T extends Array ? Array> : T extends ReadonlyArray ? ReadonlyArray> : T extends BaseType ? T : T extends { + type Unstubify = T extends StubBase ? V : T extends Map ? Map, Unstubify> : T extends Set ? Set> : T extends ReadonlyMap ? ReadonlyMap, Unstubify> : T extends ReadonlySet ? ReadonlySet> : T extends Array ? Array> : T extends ReadonlyArray ? ReadonlyArray> : T extends BaseType ? T : T extends { [key: string | number]: unknown; } ? { [K in keyof T]: Unstubify; @@ -15401,7 +15471,7 @@ declare namespace TailStream { } type EventOutcome = "ok" | "canceled" | "exception" | "unknown" | "killSwitch" | "daemonDown" | "exceededCpu" | "exceededMemory" | "loadShed" | "responseStreamDisconnected" | "scriptNotFound" | "internalError" | "exceededWallTime" | "aborted"; interface ScriptVersion { - readonly id: string; + readonly id?: string; readonly tag?: string; readonly message?: string; } @@ -15412,7 +15482,7 @@ declare namespace TailStream { } interface Onset { readonly type: "onset"; - readonly attributes: Attribute[]; + readonly attributes?: Attribute[]; // id for the span being opened by this Onset event. readonly spanId: string; readonly dispatchNamespace?: string; @@ -15551,6 +15621,7 @@ declare namespace TailStream { diagnosticChannel?: TailEventHandler; exception?: TailEventHandler; log?: TailEventHandler; + streamDiagEvent?: TailEventHandler; return?: TailEventHandler; attributes?: TailEventHandler; }; diff --git a/wrangler.jsonc b/wrangler.jsonc index ac30eea0..553bca07 100644 --- a/wrangler.jsonc +++ b/wrangler.jsonc @@ -23,6 +23,18 @@ "keep_vars": true, "vars": { "ENVIRONMENT": "production", + // Enable only after the private image bucket and B2 secrets are provisioned. + "EXTENSION_PACKAGES_MODE": "disabled", + "EXTENSION_PACKAGES_BASE_URL": "https://api.fossbilling.net/extensions/v2", + "EXTENSION_PACKAGES_S3_ENDPOINT": "https://s3.eu-central-003.backblazeb2.com", + "EXTENSION_PACKAGES_S3_REGION": "eu-central-003", + "EXTENSION_PACKAGES_S3_BUCKET": "fb-extensions-packages", + "EXTENSION_PACKAGES_S3_PREFIX": "production", + "EXTENSION_ASSETS_S3_ENDPOINT": "https://s3.eu-central-003.backblazeb2.com", + "EXTENSION_ASSETS_S3_REGION": "eu-central-003", + "EXTENSION_ASSETS_S3_BUCKET": "fb-extensions-assets", + "EXTENSION_ASSETS_S3_PREFIX": "production", + "EXTENSION_ASSETS_PUBLIC_URL": "https://extensions-assets.fossbilling.net", "EXTENSIONS_RETENTION_MODE": "dry-run", "EXTENSIONS_V2_EMAIL_PROVIDER": "mxroute", "EXTENSIONS_V2_MXROUTE_SERVER": "wednesday.mxrouting.net", @@ -60,10 +72,30 @@ "id": "0771957093ae481b9cd974ffa83f8263" } ], + "queues": { + "producers": [ + { "binding": "EXTENSION_PACKAGE_QUEUE", "queue": "extension-packages" } + ], + "consumers": [ + { + "queue": "extension-packages", + "max_batch_size": 1, + "max_batch_timeout": 1, + "max_retries": 5, + "max_concurrency": 1 + } + ] + }, + "images": { "binding": "PACKAGE_IMAGES", "remote": false }, "r2_buckets": [ { "binding": "DOWNLOAD_BUCKET", "bucket_name": "fossbilling-download" + }, + { + // Private prepared icons, including pending submissions. Provision before deployment. + "binding": "EXTENSION_IMAGE_BUCKET", + "bucket_name": "fb-extensions-private-images" } ], "services": [ @@ -101,5 +133,90 @@ "period": 60 } } - ] + ], + "env": { + "packages-local": { + "queues": { + "producers": [ + { + "binding": "EXTENSION_PACKAGE_QUEUE", + "queue": "extension-packages-local" + } + ], + "consumers": [ + { + "queue": "extension-packages-local", + "max_batch_size": 1, + "max_batch_timeout": 1, + "max_retries": 5, + "max_concurrency": 1 + } + ] + }, + "images": { "binding": "PACKAGE_IMAGES", "remote": false }, + "routes": [], + "vars": { + "ENVIRONMENT": "local", + "EXTENSIONS_RETENTION_MODE": "dry-run", + "M2M_ISSUER": "https://auth.fossbilling.net", + "M2M_JWKS_URL": "https://auth.fossbilling.net/.well-known/jwks.json", + "M2M_AUDIENCE": "https://api.fossbilling.net", + "EXTENSIONS_V2_EMAIL_PROVIDER": "disabled", + "EXTENSIONS_V2_MXROUTE_SERVER": "", + "EXTENSIONS_V2_MXROUTE_USERNAME": "", + "EXTENSION_PACKAGES_MODE": "local", + "EXTENSION_PACKAGES_BASE_URL": "http://localhost:8789/extensions/v2" + }, + "d1_databases": [ + { + "binding": "DB_CENTRAL_ALERTS", + "database_id": "00000000-0000-0000-0000-000000000001", + "migrations_dir": "src/services/central-alerts/v1/db/migrations" + }, + { + "binding": "DB_EXTENSIONS", + "database_id": "00000000-0000-0000-0000-000000000002", + "migrations_dir": "src/services/extensions/v2/db/migrations" + } + ], + "kv_namespaces": [ + { "binding": "AUTH_KV", "id": "00000000000000000000000000000001" }, + { "binding": "CACHE_KV", "id": "00000000000000000000000000000002" } + ], + "r2_buckets": [ + { "binding": "DOWNLOAD_BUCKET", "bucket_name": "downloads-local" }, + { + "binding": "EXTENSION_PACKAGE_BUCKET", + "bucket_name": "extension-packages-local" + }, + { + "binding": "EXTENSION_IMAGE_BUCKET", + "bucket_name": "extension-images-local" + } + ], + "durable_objects": { + "bindings": [ + { + "name": "PREVIEW_GITHUB_BUDGET", + "class_name": "PreviewGitHubBudget" + } + ] + }, + "services": [ + { "binding": "EXTENSIONS_FRONTEND", "service": "extensions" } + ], + "ratelimits": [ + { + "name": "EXTENSION_WRITE_RATE_LIMITER", + "namespace_id": "1001", + "simple": { "limit": 50, "period": 60 } + }, + { + "name": "PROFILE_CREATION_RATE_LIMITER", + "namespace_id": "1002", + "simple": { "limit": 3, "period": 60 } + } + ] + } + } } From 16b119fc57f8c0c6e37ea885f3fcbb8ae252b645 Mon Sep 17 00:00:00 2001 From: Adam Daley Date: Sat, 7 Nov 2026 15:21:26 +0000 Subject: [PATCH 02/12] Address package processing review findings --- src/app/index.ts | 8 +- src/services/extensions/v1/README.md | 10 +- src/services/extensions/v2/PACKAGING.md | 9 +- .../db/migrations/0030_directory_packages.sql | 12 +- .../v2/db/migrations/0031_package_jobs.sql | 4 +- .../v2/db/migrations/meta/0031_snapshot.json | 5 + src/services/extensions/v2/db/packages.ts | 21 ++- src/services/extensions/v2/db/schema.ts | 1 + src/services/extensions/v2/package-builder.ts | 23 +-- .../v2/packages/cloudflare-queue.ts | 33 +++- src/services/extensions/v2/packages/jobs.ts | 6 + .../extensions/v2/packages/public-assets.ts | 11 +- test/integration/scheduled.test.ts | 33 ++++ test/lib/context.test.ts | 8 + test/services/extensions/v1/index.test.ts | 51 ++++++ test/services/extensions/v2/db-fixtures.ts | 14 +- .../extensions/v2/package-builder.test.ts | 16 ++ .../extensions/v2/package-jobs.node.test.ts | 35 ++++- test/services/extensions/v2/packages.test.ts | 145 +++++++++++++++++- .../extensions/v2/resource-bounds.test.ts | 1 + 20 files changed, 405 insertions(+), 41 deletions(-) create mode 100644 test/integration/scheduled.test.ts diff --git a/src/app/index.ts b/src/app/index.ts index d0189fd1..4d769277 100644 --- a/src/app/index.ts +++ b/src/app/index.ts @@ -86,7 +86,13 @@ export default { _event: ScheduledController, env: CloudflareBindings ): Promise => { - await recoverPackageJobs(env); + try { + await recoverPackageJobs(env); + } catch (error) { + logError("extensions/packages", "Package job recovery failed", { + error: error instanceof Error ? error.message : String(error) + }); + } const db = getExtensionsDb(env.DB_EXTENSIONS); await maintainExtensionResources(db, { mode: env.EXTENSIONS_RETENTION_MODE diff --git a/src/services/extensions/v1/README.md b/src/services/extensions/v1/README.md index 8be81084..c48b14cc 100644 --- a/src/services/extensions/v1/README.md +++ b/src/services/extensions/v1/README.md @@ -11,11 +11,11 @@ FOSSBilling 0.8.9 still uses v1. Upgrade integrations to [extensions v2](../v2/R ## Migration -| v1 endpoint | v2 replacement | -| ------------------- | ----------------------------------------------------------------------------- | -| `GET /list` | `GET /extensions/v2/extensions` | -| `GET /{id}` | `GET /extensions/v2/extensions/{id}` | -| `GET /{id}/version` | Read the latest release's tag (`releases[0].tag`) from the v2 detail response | +| v1 endpoint | v2 replacement | +| ------------------- | ------------------------------------------------------------------------------------------------------------- | +| `GET /list` | `GET /extensions/v2/extensions` | +| `GET /{id}` | `GET /extensions/v2/extensions/{id}` | +| `GET /{id}/version` | Read `version` from the v2 detail response (the selected latest release, or newest stable release by default) | V2 lists use cursor pagination, not v1 limit/offset pagination. Use type `module` instead of `mod`. Releases carry full `requires.fossbilling` constraints; consumers must evaluate that constraint rather than a derived minimum. diff --git a/src/services/extensions/v2/PACKAGING.md b/src/services/extensions/v2/PACKAGING.md index 05050157..42daa684 100644 --- a/src/services/extensions/v2/PACKAGING.md +++ b/src/services/extensions/v2/PACKAGING.md @@ -122,6 +122,7 @@ size validation, then their original compressed payloads are copied unchanged with normalized ZIP headers. Only the generated manifest is compressed. It never runs scripts from the archive. Input is limited to 5 MiB, 500 entries and 16 MiB of actual decompressed data; multipart requests are capped at 6 MiB before parsing. +Paths are limited to 1,024 characters and 32 components. Encrypted entries, symlinks, unsafe paths, duplicate/colliding names, file/directory conflicts and corrupt archives are rejected. Checksums cover the final stored ZIP. @@ -133,8 +134,8 @@ Packaging prepares display images once in `EXTENSION_IMAGE_BUCKET`, shared by all artifacts using the same source. The original icon stays unchanged in the ZIP. SVGs are sanitized with Cloudflare svg-hush through the pinned `@zeropress/svg-hush` -Workers adapter and served directly. SVG input is capped at 2 MiB and sanitized -output at 4 MiB. Raster inputs use the local `PACKAGE_IMAGES` binding to produce +Workers adapter and served directly. SVG inputs are bounded by the ZIP expansion +limit; there is no separate SVG size cap. Raster inputs use the local `PACKAGE_IMAGES` binding to produce AVIF and PNG variants within 256 × 256 pixels, preserving proportions and transparency without enlarging small images. Requests prefer SVG when present, otherwise AVIF when explicitly accepted, with PNG fallback. They read stored @@ -170,6 +171,10 @@ Use `EXTENSION_PACKAGES_MODE=local` only outside production. The committed produ With no S3 endpoint, ZIPs use the local R2 binding. An incomplete S3 configuration or upstream failure returns 503 rather than silently switching storage backends. +R2 validates the supplied SHA-256 when writing and returns that checksum as +object metadata. Its adapter checks this metadata on reads without rehashing the +response body. The B2 adapter additionally hashes the downloaded bytes. + Source and built ZIPs use `local/packages/`. Object IDs are allocated once by D1 before upload. Uploads carry a signed SHA-256 payload and private cache headers. Reads are bounded to 17 MiB and verify artifact bytes before returning a diff --git a/src/services/extensions/v2/db/migrations/0030_directory_packages.sql b/src/services/extensions/v2/db/migrations/0030_directory_packages.sql index 33cc3ca7..2229640a 100644 --- a/src/services/extensions/v2/db/migrations/0030_directory_packages.sql +++ b/src/services/extensions/v2/db/migrations/0030_directory_packages.sql @@ -52,7 +52,7 @@ BEGIN SELECT json_extract(value,'$.artifact_id') FROM json_each(NEW.releases) ) AND source_id IN ( SELECT s.id FROM package_sources s JOIN developers d ON d.profile_generation=s.profile_generation - WHERE lower(s.extension_id)=lower(NEW.id) AND d.id=NEW.developer_id + WHERE s.extension_id=lower(NEW.id) AND d.id=NEW.developer_id ); END; @@ -80,3 +80,13 @@ CREATE TABLE `package_icons` ( --> statement-breakpoint CREATE TRIGGER package_icons_immutable BEFORE UPDATE ON package_icons BEGIN SELECT RAISE(ABORT, 'Package icons are immutable'); END; + +--> statement-breakpoint +CREATE TRIGGER package_sources_no_delete BEFORE DELETE ON package_sources +BEGIN SELECT RAISE(ABORT, 'package_source_immutable'); END; +--> statement-breakpoint +CREATE TRIGGER package_artifacts_no_delete BEFORE DELETE ON package_artifacts +BEGIN SELECT RAISE(ABORT, 'package_artifact_immutable'); END; +--> statement-breakpoint +CREATE TRIGGER package_icons_no_delete BEFORE DELETE ON package_icons +BEGIN SELECT RAISE(ABORT, 'package_icons_immutable'); END; diff --git a/src/services/extensions/v2/db/migrations/0031_package_jobs.sql b/src/services/extensions/v2/db/migrations/0031_package_jobs.sql index 70dd0f4f..8c44a8b9 100644 --- a/src/services/extensions/v2/db/migrations/0031_package_jobs.sql +++ b/src/services/extensions/v2/db/migrations/0031_package_jobs.sql @@ -25,4 +25,6 @@ CREATE TABLE `package_jobs` ( --> statement-breakpoint CREATE UNIQUE INDEX `idx_package_jobs_fingerprint` ON `package_jobs` (`fingerprint`) WHERE "package_jobs"."status" != 'failed';--> statement-breakpoint CREATE INDEX `idx_package_jobs_owner` ON `package_jobs` (`owner_user_id`);--> statement-breakpoint -CREATE INDEX `idx_package_jobs_dispatch` ON `package_jobs` (`status`,`enqueued_at`); \ No newline at end of file +CREATE INDEX `idx_package_jobs_dispatch` ON `package_jobs` (`status`,`enqueued_at`); +--> statement-breakpoint +CREATE INDEX `idx_package_jobs_upload_object` ON `package_jobs` (`upload_object_id`); diff --git a/src/services/extensions/v2/db/migrations/meta/0031_snapshot.json b/src/services/extensions/v2/db/migrations/meta/0031_snapshot.json index 08d7f6a8..af04ad9f 100644 --- a/src/services/extensions/v2/db/migrations/meta/0031_snapshot.json +++ b/src/services/extensions/v2/db/migrations/meta/0031_snapshot.json @@ -1560,6 +1560,11 @@ "name": "idx_package_jobs_dispatch", "columns": ["status", "enqueued_at"], "isUnique": false + }, + "idx_package_jobs_upload_object": { + "name": "idx_package_jobs_upload_object", + "columns": ["upload_object_id"], + "isUnique": false } }, "foreignKeys": { diff --git a/src/services/extensions/v2/db/packages.ts b/src/services/extensions/v2/db/packages.ts index 7b2fdc2e..1c333a19 100644 --- a/src/services/extensions/v2/db/packages.ts +++ b/src/services/extensions/v2/db/packages.ts @@ -113,6 +113,20 @@ export async function validatePackageContent( return fail( "The latest release needs a directory-built package; upload its source ZIP first" ); + const artifactIds = [ + ...new Set( + content.releases.flatMap((release) => + release.artifact_id ? [release.artifact_id] : [] + ) + ) + ]; + const rows = await db + .prepare( + `${ARTIFACT_SELECT} JOIN developers d ON d.profile_generation=s.profile_generation WHERE a.id IN (SELECT value FROM json_each(?)) AND s.extension_id=? AND d.id=?` + ) + .bind(JSON.stringify(artifactIds), id, developerId) + .all(); + const artifacts = new Map(rows.results.map((row) => [row.id, row])); const versions = new Set(); for (const release of content.releases) { const version = release.tag.replace(/^v/, ""); @@ -128,12 +142,7 @@ export async function validatePackageContent( ); continue; } - const artifact = await db - .prepare( - `${ARTIFACT_SELECT} JOIN developers d ON d.profile_generation=s.profile_generation WHERE a.id=? AND s.extension_id=? AND d.id=?` - ) - .bind(release.artifact_id, id, developerId) - .first(); + const artifact = artifacts.get(release.artifact_id); if ( !artifact || artifact.version !== version || diff --git a/src/services/extensions/v2/db/schema.ts b/src/services/extensions/v2/db/schema.ts index 40f29b82..b260b94b 100644 --- a/src/services/extensions/v2/db/schema.ts +++ b/src/services/extensions/v2/db/schema.ts @@ -635,6 +635,7 @@ export const packageJobs = sqliteTable( .on(table.fingerprint) .where(sql`${table.status} != 'failed'`), index("idx_package_jobs_owner").on(table.ownerUserId), + index("idx_package_jobs_upload_object").on(table.uploadObjectId), index("idx_package_jobs_dispatch").on(table.status, table.enqueuedAt), check( "package_jobs_status_check", diff --git a/src/services/extensions/v2/package-builder.ts b/src/services/extensions/v2/package-builder.ts index 2b8d18c6..ea1c0439 100644 --- a/src/services/extensions/v2/package-builder.ts +++ b/src/services/extensions/v2/package-builder.ts @@ -10,7 +10,9 @@ import { ManifestSchema } from "./schemas/manifest"; export const PACKAGE_LIMITS = { sourceBytes: 5 * 1024 * 1024, expandedBytes: 16 * 1024 * 1024, - entries: 500 + entries: 500, + pathLength: 1024, + pathDepth: 32 } as const; const zipOptions = { @@ -30,20 +32,23 @@ export async function sha256(bytes: Uint8Array) { function validatePath(filename: string, directory: boolean) { const path = directory ? filename.replace(/\/$/, "") : filename; + if (path.length > PACKAGE_LIMITS.pathLength) + throw new Error("Package path exceeds length limit"); + const parts = path.split("/"); + if (parts.length > PACKAGE_LIMITS.pathDepth) + throw new Error("Package path exceeds depth limit"); if ( !path || /[\\:<>"|?*]/.test(path) || [...path].some( (char) => char.charCodeAt(0) < 32 || char.charCodeAt(0) === 127 ) || - path.split("/").some((part) => !part || part === "." || part === "..") || - path - .split("/") - .some( - (part) => - /[. ]$/.test(part) || - /^(con|prn|aux|nul|com[1-9]|lpt[1-9])(?:\.|$)/i.test(part) - ) + parts.some((part) => !part || part === "." || part === "..") || + parts.some( + (part) => + /[. ]$/.test(part) || + /^(con|prn|aux|nul|com[1-9]|lpt[1-9])(?:\.|$)/i.test(part) + ) ) { throw new Error(`Unsafe package path: ${filename}`); } diff --git a/src/services/extensions/v2/packages/cloudflare-queue.ts b/src/services/extensions/v2/packages/cloudflare-queue.ts index fad14fc3..f7c93711 100644 --- a/src/services/extensions/v2/packages/cloudflare-queue.ts +++ b/src/services/extensions/v2/packages/cloudflare-queue.ts @@ -1,3 +1,4 @@ +import { logError } from "../../../../lib/logger"; import { createCloudflareBindings } from "../../../../lib/adapters/cloudflare"; import { CloudflareQueueAdapter } from "../../../../lib/adapters/cloudflare/queue"; import { PackageJobsDatabase } from "../db/package-jobs"; @@ -44,12 +45,20 @@ export async function prepareQueuedPackage( "Package ownership changed while queued", 403 ); - const source = job.uploadObjectId - ? await boundedBytes( - await store.get(job.uploadObjectId, job.uploadSha256 ?? undefined), - PACKAGE_LIMITS.sourceBytes - ) - : undefined; + let source: Uint8Array | undefined; + if (job.uploadObjectId) { + const body = await store.get( + job.uploadObjectId, + job.uploadSha256 ?? undefined + ); + if (!body) + throw new PackageError( + "SOURCE_UNAVAILABLE", + "Queued source ZIP is unavailable", + 503 + ); + source = await boundedBytes(body, PACKAGE_LIMITS.sourceBytes); + } const artifact = await buildStoredPackage({ db: env.DB_EXTENSIONS, store, @@ -103,6 +112,14 @@ export async function recoverPackageJobs(env: CloudflareBindings) { const jobs = new PackageJobsDatabase(env.DB_EXTENSIONS); await jobs.expire(); const queue = new CloudflareQueueAdapter(env.EXTENSION_PACKAGE_QUEUE); - for (const id of await jobs.pending()) - await dispatchPackageJob(jobs, queue, id); + for (const id of await jobs.pending()) { + try { + await dispatchPackageJob(jobs, queue, id); + } catch (error) { + logError("extensions/packages", "Package job dispatch failed", { + jobId: id, + error: error instanceof Error ? error.message : String(error) + }); + } + } } diff --git a/src/services/extensions/v2/packages/jobs.ts b/src/services/extensions/v2/packages/jobs.ts index d80c5153..99cbecec 100644 --- a/src/services/extensions/v2/packages/jobs.ts +++ b/src/services/extensions/v2/packages/jobs.ts @@ -1,3 +1,4 @@ +import { logError } from "../../../../lib/logger"; import type { IMessageQueue } from "../../../../lib/interfaces"; import type { Manifest } from "../schemas/manifest"; import type { PackageOwner } from "./build"; @@ -65,6 +66,11 @@ export async function processPackageJob( await jobs.complete(id, lease, artifactId); return "ack"; } catch (cause) { + if (!(cause instanceof PackageError)) + logError("extensions/packages", "Package preparation failed", { + jobId: id, + error: cause instanceof Error ? cause.message : String(cause) + }); const error = cause instanceof PackageError ? { code: cause.code, message: cause.message, status: cause.status } diff --git a/src/services/extensions/v2/packages/public-assets.ts b/src/services/extensions/v2/packages/public-assets.ts index 718d7cab..40d609cb 100644 --- a/src/services/extensions/v2/packages/public-assets.ts +++ b/src/services/extensions/v2/packages/public-assets.ts @@ -1,3 +1,4 @@ +import { logError } from "../../../../lib/logger"; import { revalidateExtensions } from "../revalidate"; import type { ExtensionContent } from "../schemas/extensions"; import { sha256 } from "../package-builder"; @@ -15,6 +16,7 @@ export async function publishExtensionIcon( c: Context<{ Bindings: CloudflareBindings }>, extensionId: string ) { + extensionId = extensionId.toLowerCase(); const platform = getPlatform(c); const endpoint = platform.getEnv("EXTENSION_ASSETS_S3_ENDPOINT"); if (!endpoint) return; @@ -192,10 +194,11 @@ export function scheduleIconPublication( .then((changed) => { if (changed) revalidateExtensions(c); }) - .catch(() => { - console.error( - "[packages] Icon publication failed; existing image delivery remains available" - ); + .catch((error) => { + logError("extensions/packages", "Icon publication failed", { + extensionId, + error: error instanceof Error ? error.message : String(error) + }); }) ); } diff --git a/test/integration/scheduled.test.ts b/test/integration/scheduled.test.ts new file mode 100644 index 00000000..1bf44bb9 --- /dev/null +++ b/test/integration/scheduled.test.ts @@ -0,0 +1,33 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { env } from "cloudflare:workers"; +const mocks = vi.hoisted(() => ({ + recover: vi.fn(), + maintain: vi.fn(), + report: vi.fn() +})); +vi.mock("../../src/services/extensions/v2/packages/cloudflare-queue", () => ({ + consumePackageJobs: vi.fn(), + recoverPackageJobs: mocks.recover +})); +vi.mock("../../src/services/extensions/v2/db/resource-maintenance", () => ({ + maintainExtensionResources: mocks.maintain, + reportExtensionResources: mocks.report +})); +import app from "../../src/app"; +afterEach(() => vi.restoreAllMocks()); +describe("scheduled maintenance", () => { + it("continues resource maintenance when package recovery fails", async () => { + const log = vi.spyOn(console, "error").mockImplementation(() => {}); + mocks.recover.mockRejectedValueOnce(new Error("job database unavailable")); + mocks.maintain.mockResolvedValueOnce(undefined); + mocks.report.mockResolvedValueOnce(undefined); + await app.scheduled({} as ScheduledController, env); + expect(mocks.maintain).toHaveBeenCalledOnce(); + expect(mocks.report).toHaveBeenCalledOnce(); + expect(log).toHaveBeenCalledWith( + expect.objectContaining({ + context: { error: "job database unavailable" } + }) + ); + }); +}); diff --git a/test/lib/context.test.ts b/test/lib/context.test.ts index 3a936758..86d305c9 100644 --- a/test/lib/context.test.ts +++ b/test/lib/context.test.ts @@ -9,6 +9,7 @@ import { IPlatformBindings } from "../../src/lib/interfaces"; // that boots the real app. describe("createPlatformContext", () => { const mockBindings: IPlatformBindings = { + queues: { testQueue: { send: async () => {} } }, caches: { testCache: { get: async () => null, @@ -33,9 +34,16 @@ describe("createPlatformContext", () => { expect(context.getCache("testCache")).toBe(mockBindings.caches.testCache); expect(context.getEnv("TEST_VAR")).toBe("test-value"); + expect(context.getQueue("testQueue")).toBe(mockBindings.queues!.testQueue); expect(context.raw).toBe(mockHonoContext); }); + it("throws when a queue binding is missing", () => { + const context = createPlatformContext(mockHonoContext, mockBindings); + expect(() => context.getQueue("missing")).toThrow( + "Queue binding 'missing' not found" + ); + }); it("throws when cache binding not found", () => { const context = createPlatformContext(mockHonoContext, mockBindings); diff --git a/test/services/extensions/v1/index.test.ts b/test/services/extensions/v1/index.test.ts index 94071a4a..3dff1c47 100644 --- a/test/services/extensions/v1/index.test.ts +++ b/test/services/extensions/v1/index.test.ts @@ -404,6 +404,57 @@ describe("Extensions API v1", () => { }); describe("GET /:id/version", () => { + it.each([false, true])( + "uses the selected or stable release when a newer prerelease exists (selected=%s)", + async (selected) => { + const db = getExtensionsDb(env.DB_EXTENSIONS); + const releases = [ + { + tag: "2.0.0-beta.1", + date: "2026-01-02", + download_url: "https://example.test/beta.zip", + requires: { fossbilling: "*" } + }, + { + tag: "1.0.0", + date: "2026-01-01", + download_url: "https://example.test/stable.zip", + requires: { fossbilling: "*" } + }, + { + tag: "0.9.0", + date: "2025-01-01", + download_url: "https://example.test/pinned.zip", + requires: { fossbilling: "*" }, + ...(selected ? { is_latest: true } : {}) + } + ]; + const expected = selected ? "0.9.0" : "1.0.0"; + await db + .update(extensions) + .set({ releases: JSON.stringify(releases), version: expected }) + .where(eq(extensions.id, "Example")); + const ctx = createExecutionContext(); + const response = await app.request( + "/extensions/v1/Example/version", + {}, + env, + ctx + ); + await waitOnExecutionContext(ctx); + expect(await response.text()).toBe(expected); + const detail = await app.request( + "/extensions/v1/Example", + {}, + env, + createExecutionContext() + ); + expect( + ((await detail.json()) as { result: { version: string } }).result + .version + ).toBe(expected); + } + ); it("should return plain text version", async () => { const ctx = createExecutionContext(); const res = await app.request( diff --git a/test/services/extensions/v2/db-fixtures.ts b/test/services/extensions/v2/db-fixtures.ts index cc7398e7..2dc871ab 100644 --- a/test/services/extensions/v2/db-fixtures.ts +++ b/test/services/extensions/v2/db-fixtures.ts @@ -133,12 +133,24 @@ async function clearDeveloperHistory(db: D1Database): Promise { } export async function resetExtensionsDb(db: D1Database): Promise { + // Test reset only: temporarily lift the immutable-row delete guards. + const guards = [ + ["package_sources", "package_source_immutable"], + ["package_artifacts", "package_artifact_immutable"], + ["package_icons", "package_icons_immutable"] + ]; await db.batch([ + ...guards.map(([table]) => db.prepare(`DROP TRIGGER ${table}_no_delete`)), db.prepare("DELETE FROM package_jobs"), db.prepare("DELETE FROM package_artifacts"), db.prepare("DELETE FROM package_icons"), db.prepare("DELETE FROM package_sources"), - db.prepare("DELETE FROM package_objects") + db.prepare("DELETE FROM package_objects"), + ...guards.map(([table, message]) => + db.prepare( + `CREATE TRIGGER ${table}_no_delete BEFORE DELETE ON ${table} BEGIN SELECT RAISE(ABORT, '${message}'); END` + ) + ) ]); await db.prepare("DELETE FROM extension_write_events").run(); await db.prepare("DELETE FROM claim_verification_budgets").run(); diff --git a/test/services/extensions/v2/package-builder.test.ts b/test/services/extensions/v2/package-builder.test.ts index 74a62847..381ce041 100644 --- a/test/services/extensions/v2/package-builder.test.ts +++ b/test/services/extensions/v2/package-builder.test.ts @@ -50,6 +50,22 @@ async function contents(bytes: Uint8Array) { } describe("local directory package builder", () => { + it("rejects overlong and deeply nested paths before walking ancestors", async () => { + for (const path of [ + "a/".repeat(20000) + "file.php", + "a/".repeat(PACKAGE_LIMITS.pathDepth) + "file.php", + "a".repeat(PACKAGE_LIMITS.pathLength + 1) + ]) { + await expect( + buildExtensionPackage(await zip([[path, "file"]]), manifest) + ).rejects.toThrow(/path exceeds/); + } + const path = "a/".repeat(PACKAGE_LIMITS.pathDepth - 1) + "file.php"; + expect( + (await buildExtensionPackage(await zip([[path, "file"]]), manifest)) + .sourceFileCount + ).toBe(1); + }); it("replaces the manifest and preserves every source file", async () => { const source = await zip([ ["manifest.json", "old metadata"], diff --git a/test/services/extensions/v2/package-jobs.node.test.ts b/test/services/extensions/v2/package-jobs.node.test.ts index 5782805e..818706a3 100644 --- a/test/services/extensions/v2/package-jobs.node.test.ts +++ b/test/services/extensions/v2/package-jobs.node.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, it, vi } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import { dispatchPackageJob, processPackageJob, @@ -68,6 +68,8 @@ function repository() { return { job, jobs }; } +afterEach(() => vi.restoreAllMocks()); + describe("provider-independent package job processing", () => { it("publishes only an identifier through the queue interface", async () => { const { jobs, job } = repository(); @@ -76,6 +78,28 @@ describe("provider-independent package job processing", () => { expect(send).toHaveBeenCalledWith({ jobId: job.id }); expect(jobs.markEnqueued).toHaveBeenCalledWith(job.id); }); + it("leaves an actively leased job unchanged when another delivery arrives", async () => { + const { jobs, job } = repository(); + job.status = "processing"; + const prepare = vi.fn(); + expect(await processPackageJob(jobs, job.id, prepare)).toBe("busy"); + expect(job.status).toBe("processing"); + expect(job.attempts).toBe(0); + expect(prepare).not.toHaveBeenCalled(); + }); + it("terminates jobs interrupted beyond the processing attempt limit", async () => { + const { jobs, job } = repository(); + job.attempts = 3; + const prepare = vi.fn(); + expect(await processPackageJob(jobs, job.id, prepare)).toBe("ack"); + expect(job.attempts).toBe(4); + expect(job.status).toBe("failed"); + expect(job.error).toMatchObject({ + code: "PACKAGE_UNAVAILABLE", + status: 503 + }); + expect(prepare).not.toHaveBeenCalled(); + }); it("does not run completed jobs twice", async () => { const { jobs, job } = repository(); const prepare = vi.fn().mockResolvedValue("artifact"); @@ -86,12 +110,21 @@ describe("provider-independent package job processing", () => { }); it("retries transient failures with a bounded attempt count", async () => { const { jobs, job } = repository(); + const logged = vi.spyOn(console, "error").mockImplementation(() => {}); const prepare = vi.fn().mockRejectedValue(new Error("storage unavailable")); expect(await processPackageJob(jobs, job.id, prepare)).toBe("retry"); expect(await processPackageJob(jobs, job.id, prepare)).toBe("retry"); expect(await processPackageJob(jobs, job.id, prepare)).toBe("ack"); expect(job.status).toBe("failed"); expect(job.error?.message).not.toContain("storage unavailable"); + expect(logged).toHaveBeenCalledWith( + expect.objectContaining({ + context: expect.objectContaining({ + jobId: job.id, + error: "storage unavailable" + }) + }) + ); }); it("does not retry invalid packages or revoked ownership", async () => { for (const status of [422, 403] as const) { diff --git a/test/services/extensions/v2/packages.test.ts b/test/services/extensions/v2/packages.test.ts index d416ee14..b4d0ca2d 100644 --- a/test/services/extensions/v2/packages.test.ts +++ b/test/services/extensions/v2/packages.test.ts @@ -1,6 +1,9 @@ import { PackageJobsDatabase } from "../../../../src/services/extensions/v2/db/package-jobs"; import { processPackageJob } from "../../../../src/services/extensions/v2/packages/jobs"; -import { prepareQueuedPackage } from "../../../../src/services/extensions/v2/packages/cloudflare-queue"; +import { + recoverPackageJobs, + prepareQueuedPackage +} from "../../../../src/services/extensions/v2/packages/cloudflare-queue"; import { beforeEach, afterEach, describe, expect, it, vi } from "vitest"; import { env } from "cloudflare:workers"; import { @@ -24,7 +27,10 @@ import { import { PackageArtifactSchema } from "../../../../src/services/extensions/v2/schemas/packages"; import { ManifestSchema } from "../../../../src/services/extensions/v2/schemas/manifest"; import { ExtensionContentSchema } from "../../../../src/services/extensions/v2/schemas/extensions"; -import { contentManifest } from "../../../../src/services/extensions/v2/db/packages"; +import { + validatePackageContent, + contentManifest +} from "../../../../src/services/extensions/v2/db/packages"; import { sha256 } from "../../../../src/services/extensions/v2/package-builder"; import { setupExtensionsV2Tests, @@ -147,6 +153,7 @@ async function finishJob(response: Response): Promise { `${root}/packages/${job.artifactId}`, await authHeaders("owner") ); + if (read.status !== 200) return read; return new Response(await read.text(), { status: 201, headers: { "Content-Type": "application/json" } @@ -476,6 +483,66 @@ describe("queued package preparation", () => { expect(send).toHaveBeenCalledWith({ jobId: ids[0] }); expect(await jobs.pending()).toEqual([]); }); + it("continues recovery after one queue dispatch fails", async () => { + await upload(manifest(), undefined, undefined, "owner", false); + await upload( + { ...manifest(), version: "2.0.0" }, + undefined, + undefined, + "owner", + false + ); + await db.prepare("UPDATE package_jobs SET enqueued_at=NULL").run(); + const logged = vi.spyOn(console, "error").mockImplementation(() => {}); + vi.mocked(env.EXTENSION_PACKAGE_QUEUE.send) + .mockClear() + .mockRejectedValueOnce(new Error("queue unavailable")); + await recoverPackageJobs(env); + expect(env.EXTENSION_PACKAGE_QUEUE.send).toHaveBeenCalledTimes(2); + expect(await new PackageJobsDatabase(db).pending()).toHaveLength(1); + expect(logged).toHaveBeenCalledWith( + expect.objectContaining({ + context: expect.objectContaining({ error: "queue unavailable" }) + }) + ); + }); + it("retries a missing staged upload and completes when storage recovers", async () => { + const source = await zip(); + const response = await upload( + manifest(), + source, + undefined, + "owner", + false + ); + const { result } = (await response.json()) as { result: { id: string } }; + const jobs = new PackageJobsDatabase(db); + const job = (await jobs.get(result.id))!; + await bucket().delete(job.uploadObjectId!); + expect( + await processPackageJob(jobs, job.id, (job) => + prepareQueuedPackage(env, job) + ) + ).toBe("retry"); + expect(await jobs.get(job.id)).toMatchObject({ + status: "queued", + attempts: 1, + error: { code: "SOURCE_UNAVAILABLE", status: 503 } + }); + await bucket().put(job.uploadObjectId!, source, { + sha256: job.uploadSha256! + }); + expect( + await processPackageJob(jobs, job.id, (job) => + prepareQueuedPackage(env, job) + ) + ).toBe("ack"); + expect(await jobs.get(job.id)).toMatchObject({ + status: "ready", + attempts: 2, + error: null + }); + }); it("recovers an expired processing lease without accepting stale completion", async () => { const response = await upload( manifest(), @@ -1016,6 +1083,29 @@ describe("stored directory packages", () => { "https://extensions-assets.fossbilling.net/tests-" ); }); + it("publishes an icon when a valid extension ID is supplied in uppercase", async () => { + configureB2(); + const fetch = vi.fn( + async (request: Request) => + new Response(null, { status: request.method === "HEAD" ? 404 : 200 }) + ); + vi.stubGlobal("fetch", fetch); + const artifact = await built(manifest(), await iconSource()); + const revision = await submitted(artifact); + await insertUser(db, { id: "moderator", is_moderator: 1 }); + expect( + ( + await post( + `${root}/extensions/PACKAGE-DEMO/revisions/${revision}/approve`, + await authHeaders("moderator"), + {} + ) + ).status + ).toBe(200); + expect((await getExtension(db, "package-demo"))?.icon_url).toMatch( + /^https:\/\/extensions-assets\.fossbilling\.net\// + ); + }); it("publishes AVIF and PNG with one immutable image identity for browser fallback", async () => { const remote = configureB2(); const artifact = await built(manifest(), await iconSource("png")); @@ -1210,6 +1300,57 @@ describe("stored directory packages", () => { ).rejects.toThrow("immutable"); }); + it("retains immutable package rows after job history is pruned", async () => { + const artifact = await built(); + expect((await approve(await submitted(artifact))).status).toBe(200); + await db.prepare("DELETE FROM package_jobs").run(); + for (const [table, column, id] of [ + ["package_artifacts", "id", artifact.id], + ["package_sources", "id", artifact.source_id], + ["package_icons", "source_id", artifact.source_id] + ]) + await expect( + db.prepare(`DELETE FROM ${table} WHERE ${column}=?`).bind(id).run() + ).rejects.toThrow("immutable"); + expect( + ( + await post( + `${root}/packages/cleanup`, + await authHeaders("moderator"), + {} + ) + ).status + ).toBe(200); + expect( + (await get(`${root}/packages/${artifact.id}/download`, {})).status + ).toBe(200); + }); + it("fetches artifact references together for a 100-release submission and rejects unknown artifacts", async () => { + const artifact = await built(); + const first = { ...content(artifact).releases[0], is_latest: true }; + const releases = [ + first, + ...Array.from({ length: 99 }, (_, i) => ({ + ...first, + is_latest: false, + tag: `${i + 2}.0.0`, + artifact_id: crypto.randomUUID() + })) + ]; + const prepared = vi.spyOn(db, "prepare"); + const error = await validatePackageContent( + db, + "package-demo", + "developer", + ExtensionContentSchema.parse({ ...content(artifact), releases }) + ); + expect(error?.message).toBe("Release does not match its package artifact"); + const artifactQueries = prepared.mock.calls + .map(([sql]) => sql) + .filter((sql) => sql.includes("FROM package_artifacts")); + expect(artifactQueries).toHaveLength(2); // publication state + the complete artifact set + expect(artifactQueries[1]).toContain("json_each(?)"); + }); it("revalidates the artifact at approval rather than trusting submission-time checks", async () => { const artifact = await built(); const revision = await submitted(artifact); diff --git a/test/services/extensions/v2/resource-bounds.test.ts b/test/services/extensions/v2/resource-bounds.test.ts index 6f7bc102..6d031ced 100644 --- a/test/services/extensions/v2/resource-bounds.test.ts +++ b/test/services/extensions/v2/resource-bounds.test.ts @@ -188,6 +188,7 @@ describe("Extension resource admission", () => { "releases", "website", "license", + "icon_url", "readme", "source", "version", From ae41bdbc8313ac99ba34aa241f1aedc0519a2e7c Mon Sep 17 00:00:00 2001 From: Adam Daley Date: Sat, 7 Nov 2026 19:09:52 +0000 Subject: [PATCH 03/12] Preserve local package and asset integration --- package-lock.json | 14 + package.json | 1 + scripts/package-demo.ts | 7 +- src/services/extensions/v1/database.ts | 13 +- src/services/extensions/v2/PACKAGING.md | 18 +- src/services/extensions/v2/README.md | 6 +- src/services/extensions/v2/db/badges.ts | 4 +- .../extensions/v2/db/developer-claims.ts | 180 +- .../extensions/v2/db/developer-profiles.ts | 87 +- .../extensions/v2/db/developer-transfers.ts | 16 +- src/services/extensions/v2/db/extensions.ts | 159 +- .../db/migrations/0032_directory_identity.sql | 688 ++++++ .../v2/db/migrations/meta/0032_snapshot.json | 1929 +++++++++++++++++ .../v2/db/migrations/meta/_journal.json | 7 + src/services/extensions/v2/db/packages.ts | 10 +- src/services/extensions/v2/db/revisions.ts | 46 +- src/services/extensions/v2/db/schema.ts | 37 +- src/services/extensions/v2/email/notify.ts | 4 +- .../extensions/v2/email/recipients.ts | 21 +- src/services/extensions/v2/packages/build.ts | 10 +- .../extensions/v2/packages/public-assets.ts | 25 +- src/services/extensions/v2/routes/badges.ts | 2 +- .../v2/routes/developer-profiles.ts | 6 +- .../extensions/v2/routes/moderation.ts | 35 +- .../extensions/v2/routes/owner-extensions.ts | 64 +- src/services/extensions/v2/routes/packages.ts | 11 +- .../extensions/v2/routes/public-extensions.ts | 10 +- .../extensions/v2/schemas/developers.ts | 4 +- .../extensions/v2/schemas/extensions.ts | 23 +- src/services/extensions/v2/schemas/m2m.ts | 4 +- .../extensions/v2/schemas/ownership.ts | 3 +- .../extensions/v2/schemas/packages.ts | 2 +- .../extensions/v2/schemas/revisions.ts | 3 +- test/services/extensions/v1/index.test.ts | 24 +- test/services/extensions/v2/account.test.ts | 50 +- test/services/extensions/v2/badges.test.ts | 28 +- test/services/extensions/v2/contract.test.ts | 24 +- test/services/extensions/v2/db-fixtures.ts | 89 +- .../extensions/v2/developer-approval.test.ts | 16 +- .../extensions/v2/developer-profiles.test.ts | 175 +- .../extensions/v2/directory-identity.test.ts | 95 + .../extensions/v2/extension-writes.test.ts | 93 +- test/services/extensions/v2/harness.ts | 14 +- test/services/extensions/v2/manifest.test.ts | 18 +- .../services/extensions/v2/migrations.test.ts | 65 + .../extensions/v2/moderation-correct.test.ts | 53 +- .../extensions/v2/moderation-notify.test.ts | 70 +- .../v2/moderation-revalidate.test.ts | 33 +- .../v2/moderation-revocation.test.ts | 14 +- .../services/extensions/v2/moderation.test.ts | 262 ++- test/services/extensions/v2/ownership.test.ts | 53 +- test/services/extensions/v2/packages.test.ts | 66 +- .../extensions/v2/pending-edits.test.ts | 22 +- .../extensions/v2/public-extensions.test.ts | 42 +- test/services/extensions/v2/publisher.test.ts | 50 +- .../extensions/v2/resource-bounds.test.ts | 124 +- worker-configuration.d.ts | 5 +- wrangler.jsonc | 1 - 58 files changed, 4092 insertions(+), 843 deletions(-) create mode 100644 src/services/extensions/v2/db/migrations/0032_directory_identity.sql create mode 100644 src/services/extensions/v2/db/migrations/meta/0032_snapshot.json create mode 100644 test/services/extensions/v2/directory-identity.test.ts diff --git a/package-lock.json b/package-lock.json index 49f36e78..628c5db0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -19,6 +19,7 @@ "drizzle-orm": "0.45.4", "hono": "4.13.16", "semver": "7.8.6", + "uuid": "14.0.3", "zod": "4.6.7" }, "devDependencies": { @@ -6201,6 +6202,19 @@ "punycode": "^2.1.0" } }, + "node_modules/uuid": { + "version": "14.0.3", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-14.0.3.tgz", + "integrity": "sha512-Sc5u1VVln93y1Xnz4R+CunOZc7R9LUcTmhj7Rc7Eg3HVcf7rVn2FOoa812fe0W8p3jD7xeoGSYbfqCGGH2B6xw==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "license": "MIT", + "bin": { + "uuid": "dist-node/bin/uuid" + } + }, "node_modules/verkit": { "version": "0.5.0", "resolved": "https://registry.npmjs.org/verkit/-/verkit-0.5.0.tgz", diff --git a/package.json b/package.json index 798034ff..9a753752 100644 --- a/package.json +++ b/package.json @@ -40,6 +40,7 @@ "drizzle-orm": "0.45.4", "hono": "4.13.16", "semver": "7.8.6", + "uuid": "14.0.3", "zod": "4.6.7" }, "devDependencies": { diff --git a/scripts/package-demo.ts b/scripts/package-demo.ts index e89921c5..77760716 100644 --- a/scripts/package-demo.ts +++ b/scripts/package-demo.ts @@ -1,3 +1,4 @@ +import { v7 as uuidv7 } from "uuid"; import assert from "node:assert/strict"; import { execFileSync } from "node:child_process"; import { readFile, writeFile } from "node:fs/promises"; @@ -53,8 +54,8 @@ if (action === "setup") { ('package-demo-owner','Local Package Developer',CURRENT_TIMESTAMP,CURRENT_TIMESTAMP,0), ('package-demo-reviewer','Local Package Reviewer',CURRENT_TIMESTAMP,CURRENT_TIMESTAMP,1) ON CONFLICT DO NOTHING; - INSERT INTO developers (id,type,name,owner_user_id,approved_at,created_at,updated_at) - VALUES ('package-demo-studio','user','Local Package Developer','package-demo-owner',CURRENT_TIMESTAMP,CURRENT_TIMESTAMP,CURRENT_TIMESTAMP) + INSERT INTO developers (id,identifier,type,name,owner_user_id,approved_at,created_at,updated_at) + VALUES ('${uuidv7()}','package-demo-studio','user','Local Package Developer','package-demo-owner',CURRENT_TIMESTAMP,CURRENT_TIMESTAMP,CURRENT_TIMESTAMP) ON CONFLICT DO NOTHING; ` ]); @@ -188,7 +189,7 @@ if (action === "setup") { { method: "POST", headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ id: manifest.id, ...content }) + body: JSON.stringify({ identifier: manifest.id, ...content }) }, "package-demo-owner" ); diff --git a/src/services/extensions/v1/database.ts b/src/services/extensions/v1/database.ts index f4e41730..b120c58c 100644 --- a/src/services/extensions/v1/database.ts +++ b/src/services/extensions/v1/database.ts @@ -9,9 +9,9 @@ import type { Release as ManifestRelease } from "../v2/schemas/extensions"; import { Extension, Release, Author, Repository } from "./interfaces"; const EXTENSION_COLUMNS = { - id: extensions.id, + id: extensions.identifier, type: extensions.type, - developerId: extensions.developerId, + developerId: developers.identifier, authorType: developers.type, authorName: developers.name, authorUrl: developers.url, @@ -120,7 +120,10 @@ export class ExtensionsDatabase { // the published-extension index and is unique. rows = page ? ((await base - .orderBy(sql`LOWER(${extensions.id})`, extensions.id) + .orderBy( + sql`LOWER(${extensions.identifier})`, + extensions.identifier + ) .offset(page.offset) .limit(page.limit + 1)) as ExtensionRow[]) : ((await base) as ExtensionRow[]); @@ -150,7 +153,7 @@ export class ExtensionsDatabase { .innerJoin(developers, eq(extensions.developerId, developers.id)) .where( and( - sql`LOWER(${extensions.id}) = LOWER(${id})`, + sql`LOWER(${extensions.identifier}) = LOWER(${id})`, isNotNull(extensions.publishedAt), isNull(extensions.delistedAt) ) @@ -181,7 +184,7 @@ export class ExtensionsDatabase { .from(extensions) .where( and( - sql`LOWER(${extensions.id}) = LOWER(${id})`, + sql`LOWER(${extensions.identifier}) = LOWER(${id})`, isNotNull(extensions.publishedAt), isNull(extensions.delistedAt) ) diff --git a/src/services/extensions/v2/PACKAGING.md b/src/services/extensions/v2/PACKAGING.md index 42daa684..b382bd62 100644 --- a/src/services/extensions/v2/PACKAGING.md +++ b/src/services/extensions/v2/PACKAGING.md @@ -198,13 +198,21 @@ EXTENSION_ASSETS_S3_BUCKET="fb-extensions-assets" EXTENSION_ASSETS_S3_ACCESS_KEY_ID="your-keyID" EXTENSION_ASSETS_S3_SECRET_ACCESS_KEY="your-applicationKey" EXTENSION_ASSETS_PUBLIC_URL="https://extensions-assets.fossbilling.net" -EXTENSION_ASSETS_S3_PREFIX="local" ``` -An unset endpoint disables B2 publication. Test images use `local/icons/`; -production can select a separate prefix. URLs include an immutable image identity, -and uploads set `Cache-Control: public, max-age=31536000, immutable`. Raster -identities include both variant hashes so AVIF and PNG can share a URL stem. +An unset endpoint disables B2 publication. Public object keys use +`extensions//icons/.`, for example: + +```text +https://extensions-assets.fossbilling.net/extensions/019a0000-0000-7000-8000-000000000001/icons/11111111-1111-4111-8111-111111111111.svg +``` + +The parent is the extension record's UUID, not its public identifier. The icon +record uses its immutable package-source UUID as its asset identity. All format +variants share that identity, and manifest-only edits reuse the same image URL. +A different source gets a new asset UUID. Uploads set +`Cache-Control: public, max-age=31536000, immutable` and retain each file's SHA-256 +in object metadata for verification. The API checks existing objects before uploading and caches successful checks in KV. Directory pages use direct CDN images with AVIF/PNG `` fallback, bypassing the directory image transformer. diff --git a/src/services/extensions/v2/README.md b/src/services/extensions/v2/README.md index 90b520d4..90ffd1d2 100644 --- a/src/services/extensions/v2/README.md +++ b/src/services/extensions/v2/README.md @@ -13,6 +13,10 @@ Endpoints are not listed here. The service publishes its own contract: - **OpenAPI document:** `GET /extensions/v2/docs/openapi.json` - **Reference UI:** `GET /extensions/v2/docs` +## Directory Identity + +Extensions and developers have generated UUIDv7 `id` primary keys and separate, globally unique `identifier` fields. Database relationships and machine publisher bindings use UUIDs. Public profile/extension paths use identifiers; `name` is the editable display name. Creation requests supply `identifier`, never the internal ID. The package manifest's `id` remains the public extension identifier. Staged package sources record this explicitly as `extension_identifier`, because a package can be prepared before its directory listing exists. + ## Manifest contract New v2 writes use the [FOSSBilling manifest v1 fields](https://github.com/FOSSBilling/modules/blob/main/manifest.schema.json): `manifest_version: 1`, `type: "module"` (replacing `mod`; standalone `hook` entries require manual reclassification), `author: {name, url?}`, a string `license`, optional `homepage_url` (replacing `website`), and `requires: {fossbilling, php, modules?}`. IDs allow lowercase letters, digits, underscores, and hyphens, up to 64 characters. Names are limited to 100 characters, descriptions to 500, and versions use SemVer without a `v` prefix. @@ -37,7 +41,7 @@ For example, after checking an affected package's actual requirements, an operat UPDATE extensions SET releases = json_set(releases, '$[0].requires', json('{"fossbilling":">=0.8.9","php":">=8.3"}')) -WHERE id = 'affected-extension'; +WHERE identifier = 'affected-extension'; ``` Use the ID, index, and verified constraints from the preflight investigation; the values above are examples. Re-run the query until it returns no rows. diff --git a/src/services/extensions/v2/db/badges.ts b/src/services/extensions/v2/db/badges.ts index 3f066cd6..c691a6d6 100644 --- a/src/services/extensions/v2/db/badges.ts +++ b/src/services/extensions/v2/db/badges.ts @@ -16,7 +16,7 @@ type BadgeData = { export class ExtensionBadgesDatabase { constructor(private db: ExtensionsDb) {} - async getById(id: string): Promise> { + async getByIdentifier(id: string): Promise> { try { // Select only badge fields so large README content does not affect badges. const [row] = await this.db @@ -31,7 +31,7 @@ export class ExtensionBadgesDatabase { .from(extensions) .where( and( - sql`LOWER(${extensions.id}) = LOWER(${id})`, + sql`LOWER(${extensions.identifier}) = LOWER(${id})`, isNotNull(extensions.publishedAt), isNull(extensions.delistedAt) ) diff --git a/src/services/extensions/v2/db/developer-claims.ts b/src/services/extensions/v2/db/developer-claims.ts index 0b9ba896..d9926c4b 100644 --- a/src/services/extensions/v2/db/developer-claims.ts +++ b/src/services/extensions/v2/db/developer-claims.ts @@ -1,4 +1,4 @@ -import { and, asc, desc, eq, gt, isNull, lt, or, sql, SQL } from "drizzle-orm"; +import { and, asc, desc, eq, gt, lt, or, sql, SQL } from "drizzle-orm"; import { DatabaseResult } from "../../../../lib/interfaces"; import { ExtensionsDb } from "../../../../lib/db"; import { createCursorCodec } from "./cursor"; @@ -117,123 +117,99 @@ export class DeveloperClaimsDatabase { } async claim( - developerId: string, + identifier: string, claimantId: string, note?: string, githubToken?: string ): Promise> { try { - let githubOrgVerified: number | null = null; - let githubVerificationNote: string | null = null; - const [developer] = await this.db - .select({ type: developers.type }) + .select({ + id: developers.id, + identifier: developers.identifier, + owner: developers.ownerUserId, + type: developers.type + }) .from(developers) + .where(sql`lower(${developers.identifier}) = lower(${identifier})`); + + if (!developer) + return { + data: null, + error: { code: "NOT_FOUND", message: "Developer not found" } + }; + const developerId = developer.id; + if (developer.owner !== null) + return { + data: null, + error: await this.claimIneligibilityError(developerId, claimantId) + }; + // Reject duplicates before spending the GitHub verification budget. + // The insert below rechecks ownership and claimant eligibility. + const [hasPendingClaim] = await this.db + .select({ one: sql`1` }) + .from(developerClaims) .where( - and(eq(developers.id, developerId), isNull(developers.ownerUserId)) + and( + eq(developerClaims.developerId, developerId), + eq(developerClaims.claimantId, claimantId), + eq(developerClaims.status, "pending") + ) ); - if (developer) { - // Cheap short-circuit ahead of the GitHub lookup below: a claimant - // replaying an already-pending claim on this id would otherwise - // trigger a fresh GitHub API call every time, purely to be told the - // INSERT's own guard rejects it as a duplicate — letting one caller - // burn through the shared service-level GitHub quota for free. This - // is safe precisely because it only ever *returns* here when the - // read observes `pending` — it never falls through to verification - // or the INSERT in that case, so it can't itself create an - // unverified claim. Anything else (no claim yet, or one already - // resolved to approved/rejected) always continues through full - // verification below. A pending claim that resolves between this - // read and the response going out can make the message stale - // relative to that instant, but never lets a row get created - // without verification. - const [hasPendingClaim] = await this.db - .select({ one: sql`1` }) - .from(developerClaims) - .where( - and( - eq(developerClaims.developerId, developerId), - eq(developerClaims.claimantId, claimantId), - eq(developerClaims.status, "pending") - ) - ); - - if (hasPendingClaim) { - return { - data: null, - error: { - code: "CONFLICT", - message: "You already have a pending claim on this profile" - } - }; - } - - if ( - !(await reserveClaimVerification(this.db, claimantId, developerId)) - ) { - return { - data: null, - error: { - code: "RATE_LIMITED", - message: "Claim verification allowance exhausted; try again later" - } - }; - } + if (hasPendingClaim) { + return { + data: null, + error: { + code: "CONFLICT", + message: "You already have a pending claim on this profile" + } + }; + } - const check = await verifyGithubOwnership( - this.db, - developerId, - developer.type as Developer["type"], - claimantId, - githubToken - ); + if (!(await reserveClaimVerification(this.db, claimantId, developerId))) { + return { + data: null, + error: { + code: "RATE_LIMITED", + message: "Claim verification allowance exhausted; try again later" + } + }; + } - if ("error" in check) { - return { data: null, error: check.error }; - } + const check = await verifyGithubOwnership( + this.db, + developer.identifier, + developer.type as Developer["type"], + claimantId, + githubToken + ); - if (check.mismatch) { - return { - data: null, - error: { - code: "GITHUB_MISMATCH", - message: - "Your linked GitHub account doesn't match this developer's GitHub organization or username, so it can't be claimed automatically. Make sure you're signed in with the right GitHub account, then try again." - } - }; - } + if ("error" in check) { + return { data: null, error: check.error }; + } - githubOrgVerified = check.githubOrgVerified; - githubVerificationNote = check.note; + if (check.mismatch) { + return { + data: null, + error: { + code: "GITHUB_MISMATCH", + message: + "Your linked GitHub account doesn't match this developer's GitHub organization or username, so it can't be claimed automatically. Make sure you're signed in with the right GitHub account, then try again." + } + }; } const id = crypto.randomUUID(); - let result; - try { - // Both eligibility checks are folded into the INSERT itself, rather - // than a separate SELECT beforehand — a caller who loses eligibility - // (developer gets claimed/transferred, or the caller picks up a - // different profile) between an up-front check and the write could - // otherwise still slip a stale claim through. (The SELECT above is - // only used to decide the GitHub verification signal, and is always - // re-checked here — it can't itself grant eligibility.) Kept as raw - // sql: an INSERT...SELECT...WHERE EXISTS isn't expressible via - // .insert().values(). - // ON CONFLICT DO NOTHING turns a lost race against the pending-claim - // partial unique index into changes = 0, which the ineligibility - // diagnosis below already has to explain. - result = await this.db.run(sql` - INSERT INTO ${developerClaims} (id, developer_id, claimant_id, note, github_org_verified, github_verification_note) - SELECT ${id}, ${developerId}, ${claimantId}, ${note ?? null}, ${githubOrgVerified}, ${githubVerificationNote} - WHERE EXISTS (SELECT 1 FROM ${developers} WHERE id = ${developerId} AND owner_user_id IS NULL) - AND NOT EXISTS (SELECT 1 FROM ${developers} WHERE owner_user_id = ${claimantId}) - AND EXISTS (SELECT 1 FROM ${users} WHERE ${users.id} = ${claimantId} AND ${users.deletedAt} IS NULL) - ON CONFLICT DO NOTHING - `); - } catch (error) { - return databaseError("claim", error); - } + // Ownership can change during verification; keep eligibility checks in the write. + const result = await this.db.run(sql` + INSERT INTO ${developerClaims} (id, developer_id, claimant_id, note, github_org_verified, github_verification_note) + SELECT ${id}, ${developerId}, ${claimantId}, ${note ?? null}, ${check.githubOrgVerified}, ${check.note} + WHERE EXISTS (SELECT 1 FROM ${developers} WHERE id = ${developerId} AND owner_user_id IS NULL) + AND NOT EXISTS (SELECT 1 FROM ${developers} WHERE owner_user_id = ${claimantId}) + AND EXISTS (SELECT 1 FROM ${users} WHERE ${users.id} = ${claimantId} AND ${users.deletedAt} IS NULL) + ON CONFLICT DO NOTHING + `); if (!result.meta?.changes) { return { @@ -379,6 +355,7 @@ export class DeveloperClaimsDatabase { const rows = await this.db .select({ claim: developerClaims, + developerIdentifier: developers.identifier, developerName: developers.name, developerType: developers.type, claimantName: users.name, @@ -406,6 +383,7 @@ export class DeveloperClaimsDatabase { data: { items: pageRows.map((row) => ({ ...parseClaimRow(row.claim), + developer_identifier: row.developerIdentifier, developer_name: row.developerName, developer_type: row.developerType as PendingDeveloperClaim["developer_type"], diff --git a/src/services/extensions/v2/db/developer-profiles.ts b/src/services/extensions/v2/db/developer-profiles.ts index ac4c3ecf..336a4c0b 100644 --- a/src/services/extensions/v2/db/developer-profiles.ts +++ b/src/services/extensions/v2/db/developer-profiles.ts @@ -1,3 +1,4 @@ +import { v7 as uuidv7 } from "uuid"; import { and, asc, desc, eq, gt, isNull, lt, or, sql, SQL } from "drizzle-orm"; import { DatabaseResult } from "../../../../lib/interfaces"; import { ExtensionsDb } from "../../../../lib/db"; @@ -60,6 +61,7 @@ type DeveloperRow = typeof developers.$inferSelect; function parseDeveloperRow(row: DeveloperRow): DeveloperProfile { return { id: row.id, + identifier: row.identifier, type: row.type as DeveloperProfile["type"], name: row.name, URL: row.url ?? undefined, @@ -157,7 +159,7 @@ export class DeveloperProfilesDatabase { } // githubToken — see the comment on verifyGithubOwnership(). Only consulted - // when creating a brand-new profile (developer.id is immutable once + // when creating a brand-new profile (the identifier is immutable once // owned, so an update can't need re-verifying); guards against squatting // on an id that matches a real GitHub org/user the caller doesn't control, // the one gap claim() alone can't close since it only ever applies to @@ -177,9 +179,15 @@ export class DeveloperProfilesDatabase { .select() .from(developers) .where(eq(developers.ownerUserId, userId)), - this.db.select().from(developers).where(eq(developers.id, developer.id)) + this.db + .select() + .from(developers) + .where( + sql`lower(${developers.identifier}) = lower(${developer.identifier})` + ) ]); const isCreating = !existingOwn; + const developerId = existingOwn?.id ?? uuidv7(); let githubOrgVerified: number | null = null; let githubUrlVerified: number | null = null; @@ -219,7 +227,7 @@ export class DeveloperProfilesDatabase { const check = await verifyGithubOwnership( this.db, - developer.id, + developer.identifier, developer.type, userId, githubToken, @@ -250,13 +258,13 @@ export class DeveloperProfilesDatabase { // a deletion can win between that check and this statement. mainStmt = toD1Statement(this.db.$client, { sql: `INSERT INTO developers ( - id, type, name, url, avatar_url, contact_email, + id, identifier, type, name, url, avatar_url, contact_email, owner_user_id, approved_at, created_at, updated_at, github_org_verified, github_verification_note, github_verified_at, github_url_verified ) - SELECT ?, ?, ?, ?, ?, ?, ?, NULL, CURRENT_TIMESTAMP, + SELECT ?, ?, ?, ?, ?, ?, ?, ?, NULL, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, ?, ?, CASE WHEN ? IS NULL THEN NULL ELSE CURRENT_TIMESTAMP END, ? @@ -271,7 +279,8 @@ export class DeveloperProfilesDatabase { ) ON CONFLICT DO NOTHING`, params: [ - developer.id, + developerId, + developer.identifier, developer.type, developer.name, developer.URL ?? null, @@ -288,7 +297,7 @@ export class DeveloperProfilesDatabase { ] }); } else { - if (developer.id !== existingOwn.id) { + if (developer.identifier !== existingOwn.identifier) { return { data: null, error: { @@ -336,7 +345,7 @@ export class DeveloperProfilesDatabase { }) .where( and( - eq(developers.id, developer.id), + eq(developers.id, developerId), eq(developers.ownerUserId, userId), // Pin approval/verification decisions to the version read above. eq(developers.contentRevision, existingOwn.contentRevision), @@ -373,7 +382,7 @@ export class DeveloperProfilesDatabase { WHERE changes() = 1`, params: [ crypto.randomUUID(), - developer.id, + developerId, developer.type, developer.name, developer.URL ?? null, @@ -396,7 +405,7 @@ export class DeveloperProfilesDatabase { .from(developers) .where( and( - eq(developers.id, developer.id), + eq(developers.id, developerId), eq(developers.ownerUserId, userId), sql`EXISTS (SELECT 1 FROM ${users} WHERE ${users.id} = ${userId} AND ${users.deletedAt} IS NULL)` @@ -411,7 +420,11 @@ export class DeveloperProfilesDatabase { } return { data: null, - error: await this.upsertBlockedError(userId, developer.id, isCreating) + error: await this.upsertBlockedError( + userId, + developer.identifier, + isCreating + ) }; } @@ -420,7 +433,7 @@ export class DeveloperProfilesDatabase { .from(developers) .where( and( - eq(developers.id, developer.id), + eq(developers.id, developerId), eq(developers.ownerUserId, userId) ) ); @@ -462,7 +475,7 @@ export class DeveloperProfilesDatabase { this.db .select({ id: developers.id }) .from(developers) - .where(eq(developers.id, developerId)), + .where(sql`lower(${developers.identifier}) = lower(${developerId})`), this.db .select({ id: developers.id }) .from(developers) @@ -669,19 +682,26 @@ export class DeveloperProfilesDatabase { } } - async getById( - id: string + getById(id: string) { + return this.getProfile(eq(developers.id, id)); + } + + getByIdentifier(identifier: string) { + return this.getProfile( + sql`lower(${developers.identifier}) = lower(${identifier})` + ); + } + + private async getProfile( + condition: SQL ): Promise> { try { - const [row] = await this.db - .select() - .from(developers) - .where(eq(developers.id, id)); + const [row] = await this.db.select().from(developers).where(condition); if (!row) { return { data: null, error: { - message: `Cannot find developer by id: ${id}`, + message: "Developer not found", code: "NOT_FOUND" } }; @@ -828,7 +848,9 @@ export class DeveloperProfilesDatabase { expectedRevision: number, expectedGeneration: string, reviewerId: string - ): Promise> { + ): Promise< + DatabaseResult<{ id: string; identifier: string; approved: true }> + > { let result; try { result = await this.db @@ -840,7 +862,7 @@ export class DeveloperProfilesDatabase { }) .where( and( - eq(developers.id, id), + sql`lower(${developers.identifier}) = lower(${id})`, eq(developers.contentRevision, expectedRevision), eq(developers.profileGeneration, expectedGeneration), sql`EXISTS ( @@ -849,16 +871,17 @@ export class DeveloperProfilesDatabase { AND ${users.isModerator} = 1 )` ) - ); + ) + .returning({ id: developers.id, identifier: developers.identifier }); } catch (error) { return databaseError("approve", error); } - if (!result.meta?.changes) { + if (!result[0]) { const actorError = await moderatorActorError(this.db, reviewerId); if (actorError) return { data: null, error: actorError }; - const existing = await this.getById(id); + const existing = await this.getByIdentifier(id); if (existing.error) { // A failed diagnostic lookup is not evidence that the developer is // missing. Preserve database errors (and any other lookup error) so @@ -876,7 +899,7 @@ export class DeveloperProfilesDatabase { }; } - return { data: { id, approved: true }, error: null }; + return { data: { ...result[0], approved: true }, error: null }; } // Newest-first keyset pages of one developer's audit history. @@ -919,7 +942,9 @@ export class DeveloperProfilesDatabase { let rows; try { - const conditions = [eq(developerHistory.developerId, developerId)]; + const conditions = [ + sql`${developerHistory.developerId} = (SELECT id FROM developers WHERE identifier=${developerId})` + ]; if (decoded) { conditions.push( or( @@ -1006,6 +1031,7 @@ export class DeveloperProfilesDatabase { const [row] = await this.db .select({ id: developers.id, + identifier: developers.identifier, type: developers.type, url: developers.url }) @@ -1102,7 +1128,7 @@ export class DeveloperProfilesDatabase { const matches = matchesClaimant( row.type as Developer["type"], - row.id, + row.identifier, identity.data ); @@ -1124,7 +1150,10 @@ export class DeveloperProfilesDatabase { if (!matches) { writeUrlVerified = true; } else if (checkUrl) { - const entity = await checkGithubEntity(row.id, githubToken ?? ""); + const entity = await checkGithubEntity( + row.identifier, + githubToken ?? "" + ); // An unavailable lookup is explicitly inconclusive, not a disproof, // so it leaves the stored URL verification signal untouched rather // than clearing a real prior verification over a transient failure. diff --git a/src/services/extensions/v2/db/developer-transfers.ts b/src/services/extensions/v2/db/developer-transfers.ts index df042404..b8d9457b 100644 --- a/src/services/extensions/v2/db/developer-transfers.ts +++ b/src/services/extensions/v2/db/developer-transfers.ts @@ -58,9 +58,15 @@ export class DeveloperTransfersDatabase { } async initiateTransfer( - developerId: string, + identifier: string, userId: string ): Promise> { + const profile = await new DeveloperProfilesDatabase( + this.db + ).getByIdentifier(identifier); + if (profile.error || !profile.data) + return { data: null, error: profile.error }; + const developerId = profile.data.id; try { const token = crypto.randomUUID().replace(/-/g, "") + @@ -140,9 +146,15 @@ export class DeveloperTransfersDatabase { } async revokeTransfer( - developerId: string, + identifier: string, userId: string ): Promise> { + const profile = await new DeveloperProfilesDatabase( + this.db + ).getByIdentifier(identifier); + if (profile.error || !profile.data) + return { data: null, error: profile.error }; + const developerId = profile.data.id; try { const result = await this.db.run(sql` UPDATE ${developerTransfers} SET revoked_at = CURRENT_TIMESTAMP diff --git a/src/services/extensions/v2/db/extensions.ts b/src/services/extensions/v2/db/extensions.ts index 2bcfef70..1544d19d 100644 --- a/src/services/extensions/v2/db/extensions.ts +++ b/src/services/extensions/v2/db/extensions.ts @@ -1,3 +1,4 @@ +import { v7 as uuidv7 } from "uuid"; import { packageIconUrl } from "./packages"; import { validatePackageContent } from "./packages"; import { MAX_CONTENT_BYTES } from "../resource-limits"; @@ -39,6 +40,7 @@ export const MAX_PENDING_REVISIONS_PER_USER = 10; // carry a dangling one through its rebuild. const DEVELOPER_COLUMNS = { developerId: developers.id, + developerIdentifier: developers.identifier, developerType: developers.type, developerName: developers.name, developerUrl: developers.url, @@ -137,6 +139,7 @@ const { const EXTENSION_COLUMNS = { id: extensions.id, + identifier: extensions.identifier, ...DETAIL_CONTENT_COLUMNS, publishedBytes, ...DEVELOPER_COLUMNS @@ -144,6 +147,7 @@ const EXTENSION_COLUMNS = { const EXTENSION_LIST_COLUMNS = { id: extensions.id, + identifier: extensions.identifier, ...CARD_CONTENT_COLUMNS, publishedBytes, ...DEVELOPER_COLUMNS @@ -191,6 +195,7 @@ const REVIEW_COLUMNS = { // and the pending revision's stored content (up to 256 KiB per row) with it. const OWNED_LIST_COLUMNS = { id: extensions.id, + identifier: extensions.identifier, publishedAt: extensions.publishedAt, delistedAt: extensions.delistedAt, delistReason: sql`substr(${extensions.delistReason}, 1, 2000)`, @@ -236,6 +241,7 @@ type OwnedRow = Awaited>[number]; interface DeveloperRow { developerId: string; + developerIdentifier: string; developerType: string; developerName: string; developerUrl: string | null; @@ -254,6 +260,7 @@ interface DeveloperRow { interface PublishedRow extends DeveloperRow { manifest: string | null; id: string; + identifier: string; type: string; name: string; description: string; @@ -275,7 +282,7 @@ type PublishedListRow = Omit< export interface ExtensionListFilters { type?: string; - developerId?: string; + developerIdentifier?: string; limit?: number; cursor?: string; } @@ -293,22 +300,18 @@ export interface OwnedExtensionListPage { } type ExtensionCursor = { - normalizedId: string; + identifier: string; id: string; }; -// normalizedId is checked against id rather than trusted: it drives the -// keyset comparison, so a tampered cursor could otherwise seek from a -// position the id itself doesn't correspond to. +// Public identifiers order the directory; the UUID breaks ties. const extensionCursor = createCursorCodec( (parsed): parsed is ExtensionCursor => - typeof parsed.id === "string" && - typeof parsed.normalizedId === "string" && - parsed.normalizedId === parsed.id.toLowerCase() + typeof parsed.id === "string" && typeof parsed.identifier === "string" ); export interface CreateExtensionInput { - extensionId: string; + identifier: string; developerId: string; ownershipEpoch: number; submittedBy: string; @@ -320,7 +323,7 @@ export interface CreateExtensionInput { // route's mapping lookup is only advisory; the insert re-derives it), and // submittedBy is the client's synthetic users row. export interface CreateExtensionAsPublisherInput { - extensionId: string; + identifier: string; clientId: string; developerId: string; submittedBy: string; @@ -340,8 +343,10 @@ export class ExtensionsDatabase { isNull(extensions.delistedAt) ]; if (filters.type) conditions.push(eq(extensions.type, filters.type)); - if (filters.developerId) - conditions.push(eq(extensions.developerId, filters.developerId)); + if (filters.developerIdentifier) + conditions.push( + sql`lower(${developers.identifier}) = lower(${filters.developerIdentifier})` + ); if (filters.cursor) { const cursor = extensionCursor.decode(filters.cursor); @@ -356,7 +361,7 @@ export class ExtensionsDatabase { .from(extensions) .innerJoin(developers, eq(extensions.developerId, developers.id)) .where(and(...conditions)) - .orderBy(asc(sql`LOWER(${extensions.id})`), asc(extensions.id)) + .orderBy(asc(sql`LOWER(${extensions.identifier})`), asc(extensions.id)) .limit(limit + 1)) as PublishedListRow[]; } catch (error) { return databaseError("list", error); @@ -373,7 +378,7 @@ export class ExtensionsDatabase { }; } - async getById(id: string): Promise> { + async getByIdentifier(id: string): Promise> { let rows: PublishedRow[]; try { rows = (await this.db @@ -382,7 +387,7 @@ export class ExtensionsDatabase { .innerJoin(developers, eq(extensions.developerId, developers.id)) .where( and( - sql`LOWER(${extensions.id}) = LOWER(${id})`, + sql`LOWER(${extensions.identifier}) = LOWER(${id})`, isNotNull(extensions.publishedAt), isNull(extensions.delistedAt) ) @@ -433,7 +438,7 @@ export class ExtensionsDatabase { try { rows = await ownedListQuery(this.db) .where(and(...conditions)) - .orderBy(asc(sql`LOWER(${extensions.id})`), asc(extensions.id)) + .orderBy(asc(sql`LOWER(${extensions.identifier})`), asc(extensions.id)) .limit(limit + 1); } catch (error) { return databaseError("listOwned", error); @@ -483,7 +488,7 @@ export class ExtensionsDatabase { // produces for a non-ASCII id. const pattern = `%${escapeLikePattern(filters.q)}%`; conditions.push( - sql`LOWER(${extensions.id}) LIKE LOWER(${pattern}) ESCAPE '\\'` + sql`LOWER(${extensions.identifier}) LIKE LOWER(${pattern}) ESCAPE '\\'` ); } if (filters.type) { @@ -505,7 +510,7 @@ export class ExtensionsDatabase { try { const query = ownedListQuery(this.db); rows = await (conditions.length ? query.where(and(...conditions)) : query) - .orderBy(asc(sql`LOWER(${extensions.id})`), asc(extensions.id)) + .orderBy(asc(sql`LOWER(${extensions.identifier})`), asc(extensions.id)) .limit(limit + 1); } catch (error) { return databaseError("listForModeration", error); @@ -576,7 +581,7 @@ export class ExtensionsDatabase { }) .from(extensions) .innerJoin(developers, eq(extensions.developerId, developers.id)) - .where(sql`LOWER(${extensions.id}) = LOWER(${id})`); + .where(sql`LOWER(${extensions.identifier}) = LOWER(${id})`); const row = rows[0]; if (!row) return notFound(id); @@ -599,7 +604,7 @@ export class ExtensionsDatabase { let rows: OwnedRow[]; try { rows = await ownedQuery(this.db).where( - sql`LOWER(${extensions.id}) = LOWER(${id})` + sql`LOWER(${extensions.identifier}) = LOWER(${id})` ); } catch (error) { return databaseError("getOwned", error); @@ -633,21 +638,24 @@ export class ExtensionsDatabase { // toD1Statement for why this is raw sql rather than two builder calls. async create( input: CreateExtensionInput - ): Promise> { + ): Promise< + DatabaseResult<{ id: string; identifier: string; revisionId: string }> + > { const revisionId = crypto.randomUUID(); + const extensionId = uuidv7(); let results; try { const packageError = await validatePackageContent( this.db.$client, - input.extensionId, + input.identifier, input.developerId, input.content ); if (packageError) return { data: null, error: packageError }; const extensionStmt = toD1Statement(this.db.$client, { - sql: `INSERT INTO extensions (id, developer_id, created_by, created_at, updated_at) - SELECT ?, d.id, d.owner_user_id, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP + sql: `INSERT INTO extensions (id, identifier, developer_id, created_by, created_at, updated_at) + SELECT ?, ?, d.id, d.owner_user_id, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP FROM developers d WHERE d.id = ? AND d.owner_user_id = ? AND d.ownership_epoch = ? AND EXISTS ( @@ -659,7 +667,8 @@ export class ExtensionsDatabase { ) < ? ON CONFLICT DO NOTHING`, params: [ - input.extensionId, + extensionId, + input.identifier, input.developerId, input.submittedBy, input.ownershipEpoch, @@ -676,7 +685,7 @@ export class ExtensionsDatabase { WHERE changes() = 1`, params: [ revisionId, - input.extensionId, + extensionId, input.developerId, input.submittedBy, JSON.stringify(input.content), @@ -697,7 +706,10 @@ export class ExtensionsDatabase { } } - return { data: { id: input.extensionId, revisionId }, error: null }; + return { + data: { id: extensionId, identifier: input.identifier, revisionId }, + error: null + }; } // The insert affected no rows: either ON CONFLICT DO NOTHING swallowed an id @@ -712,7 +724,7 @@ export class ExtensionsDatabase { const [taken] = await this.db .select({ one: sql`1` }) .from(extensions) - .where(sql`LOWER(${extensions.id}) = LOWER(${input.extensionId})`); + .where(sql`LOWER(${extensions.identifier}) = LOWER(${input.identifier})`); if (taken) { return { message: "An extension with this id already exists", @@ -732,21 +744,24 @@ export class ExtensionsDatabase { // users row); the mapped developer's human owner must still be active. async createAsPublisher( input: CreateExtensionAsPublisherInput - ): Promise> { + ): Promise< + DatabaseResult<{ id: string; identifier: string; revisionId: string }> + > { const revisionId = crypto.randomUUID(); + const extensionId = uuidv7(); let results; try { const packageError = await validatePackageContent( this.db.$client, - input.extensionId, + input.identifier, input.developerId, input.content ); if (packageError) return { data: null, error: packageError }; const extensionStmt = toD1Statement(this.db.$client, { - sql: `INSERT INTO extensions (id, developer_id, created_by, created_at, updated_at) - SELECT ?, d.id, d.owner_user_id, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP + sql: `INSERT INTO extensions (id, identifier, developer_id, created_by, created_at, updated_at) + SELECT ?, ?, d.id, d.owner_user_id, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP FROM developers d WHERE d.id = ? AND EXISTS ( @@ -763,7 +778,8 @@ export class ExtensionsDatabase { ) < ? ON CONFLICT DO NOTHING`, params: [ - input.extensionId, + extensionId, + input.identifier, input.developerId, input.clientId, input.submittedBy, @@ -780,7 +796,7 @@ export class ExtensionsDatabase { AND changes() = 1`, params: [ revisionId, - input.extensionId, + extensionId, input.submittedBy, JSON.stringify(input.content), input.developerId @@ -803,7 +819,10 @@ export class ExtensionsDatabase { } } - return { data: { id: input.extensionId, revisionId }, error: null }; + return { + data: { id: extensionId, identifier: input.identifier, revisionId }, + error: null + }; } private async createAsPublisherBlockedError( @@ -863,7 +882,7 @@ export class ExtensionsDatabase { const [taken] = await this.db .select({ one: sql`1` }) .from(extensions) - .where(sql`LOWER(${extensions.id}) = LOWER(${input.extensionId})`); + .where(sql`LOWER(${extensions.identifier}) = LOWER(${input.identifier})`); if (taken) { return { message: "An extension with this id already exists", @@ -883,12 +902,12 @@ export class ExtensionsDatabase { async withdraw( id: string, ownerUserId: string - ): Promise> { + ): Promise> { let result; try { - result = await this.db.run(sql` + result = await this.db.all<{ id: string; identifier: string }>(sql` DELETE FROM ${extensions} - WHERE LOWER(id) = LOWER(${id}) + WHERE LOWER(identifier) = LOWER(${id}) AND published_at IS NULL AND developer_id IN ( SELECT d.id FROM ${developers} d WHERE d.owner_user_id = ${ownerUserId} @@ -897,16 +916,17 @@ export class ExtensionsDatabase { SELECT 1 FROM ${users} u WHERE u.id = ${ownerUserId} AND u.deleted_at IS NULL ) + RETURNING id,identifier `); } catch (error) { return databaseError("withdraw", error); } - if (!result.meta?.changes) { + if (!result[0]) { return this.withdrawBlockedError(id, ownerUserId); } - return { data: { id }, error: null }; + return { data: result[0], error: null }; } // Separates the four ways the delete can affect no rows, so the route can @@ -928,7 +948,7 @@ export class ExtensionsDatabase { }) .from(extensions) .innerJoin(developers, eq(extensions.developerId, developers.id)) - .where(sql`LOWER(${extensions.id}) = LOWER(${id})`); + .where(sql`LOWER(${extensions.identifier}) = LOWER(${id})`); if (!existing) return notFound(id); if (existing.publishedAt) { return { @@ -966,7 +986,7 @@ export class ExtensionsDatabase { id: string, moderatorId: string, reason: string - ): Promise> { + ): Promise> { let result; try { result = await this.db @@ -978,7 +998,7 @@ export class ExtensionsDatabase { }) .where( and( - sql`LOWER(${extensions.id}) = LOWER(${id})`, + sql`LOWER(${extensions.identifier}) = LOWER(${id})`, isNotNull(extensions.publishedAt), isNull(extensions.delistedAt), sql`EXISTS ( @@ -987,16 +1007,17 @@ export class ExtensionsDatabase { AND ${users.isModerator} = 1 )` ) - ); + ) + .returning({ id: extensions.id, identifier: extensions.identifier }); } catch (error) { return databaseError("delist", error); } - if (!result.meta?.changes) { + if (!result[0]) { return this.delistBlockedError(id, moderatorId); } - return { data: { id }, error: null }; + return { data: result[0], error: null }; } // Separates the three ways delist()'s guard can affect no rows, so the @@ -1017,7 +1038,7 @@ export class ExtensionsDatabase { delistedAt: extensions.delistedAt }) .from(extensions) - .where(sql`LOWER(${extensions.id}) = LOWER(${id})`); + .where(sql`LOWER(${extensions.identifier}) = LOWER(${id})`); } catch (error) { return databaseError("delist", error); } @@ -1057,7 +1078,7 @@ export class ExtensionsDatabase { async relist( id: string, moderatorId: string - ): Promise> { + ): Promise> { let rows; try { rows = await this.db @@ -1069,7 +1090,7 @@ export class ExtensionsDatabase { }) .where( and( - sql`LOWER(${extensions.id}) = LOWER(${id})`, + sql`LOWER(${extensions.identifier}) = LOWER(${id})`, isNotNull(extensions.publishedAt), isNotNull(extensions.delistedAt), sql`EXISTS ( @@ -1080,7 +1101,7 @@ export class ExtensionsDatabase { )` ) ) - .returning({ id: extensions.id }); + .returning({ id: extensions.id, identifier: extensions.identifier }); } catch (error) { return databaseError("relist", error); } @@ -1090,7 +1111,7 @@ export class ExtensionsDatabase { return this.relistBlockedError(id, moderatorId); } - return { data: { id: row.id }, error: null }; + return { data: row, error: null }; } private async relistBlockedError( @@ -1113,7 +1134,7 @@ export class ExtensionsDatabase { delistedAt: extensions.delistedAt }) .from(extensions) - .where(sql`LOWER(${extensions.id}) = LOWER(${id})`); + .where(sql`LOWER(${extensions.identifier}) = LOWER(${id})`); } catch (error) { return databaseError("relist", error); } @@ -1155,7 +1176,9 @@ export class ExtensionsDatabase { moderatorId: string, content: ExtensionContent, correctionNote: string - ): Promise> { + ): Promise< + DatabaseResult<{ id: string; identifier: string; revisionId: string }> + > { const parsed = ExtensionContentSchema.safeParse(content); if (!parsed.success) { return { @@ -1185,7 +1208,7 @@ export class ExtensionsDatabase { SELECT ?, e.id, e.developer_id, ?, 'approved', ?, ?, ?, CURRENT_TIMESTAMP, d.ownership_epoch FROM extensions e JOIN developers d ON d.id = e.developer_id - WHERE LOWER(e.id) = LOWER(?) + WHERE LOWER(e.identifier) = LOWER(?) AND e.published_at IS NOT NULL AND e.delisted_at IS NULL AND NOT EXISTS ( @@ -1196,7 +1219,7 @@ export class ExtensionsDatabase { SELECT 1 FROM users u WHERE u.id = ? AND u.deleted_at IS NULL AND u.is_moderator = 1 ) - RETURNING extension_id`, + RETURNING extension_id, (SELECT identifier FROM extensions WHERE id=extension_id) AS identifier`, params: [ revisionId, moderatorId, @@ -1220,7 +1243,7 @@ export class ExtensionsDatabase { download_url = ?, manifest = ?, published_revision_id = ?, updated_at = CURRENT_TIMESTAMP - WHERE changes() = 1 AND LOWER(id) = LOWER(?)`, + WHERE changes() = 1 AND LOWER(identifier) = LOWER(?)`, params: [ valid.type, valid.name, @@ -1251,6 +1274,8 @@ export class ExtensionsDatabase { return { data: { id: (results[0].results[0] as { extension_id: string }).extension_id, + identifier: (results[0].results[0] as { identifier: string }) + .identifier, revisionId }, error: null @@ -1277,7 +1302,7 @@ export class ExtensionsDatabase { delistedAt: extensions.delistedAt }) .from(extensions) - .where(sql`LOWER(${extensions.id}) = LOWER(${id})`); + .where(sql`LOWER(${extensions.identifier}) = LOWER(${id})`); } catch (error) { return databaseError("moderatorCorrect", error); } @@ -1307,7 +1332,7 @@ export class ExtensionsDatabase { .from(extensionRevisions) .where( and( - sql`LOWER(${extensionRevisions.extensionId}) = LOWER(${id})`, + sql`${extensionRevisions.extensionId} = (SELECT id FROM extensions WHERE lower(identifier)=lower(${id}))`, eq(extensionRevisions.status, "pending") ) ); @@ -1353,9 +1378,9 @@ function notFound(id: string): DatabaseResult { function keysetAfter(cursor: ExtensionCursor) { return or( - sql`LOWER(${extensions.id}) > ${cursor.normalizedId}`, + sql`LOWER(${extensions.identifier}) > ${cursor.identifier}`, and( - sql`LOWER(${extensions.id}) = ${cursor.normalizedId}`, + sql`LOWER(${extensions.identifier}) = ${cursor.identifier}`, sql`${extensions.id} > ${cursor.id}` ) )!; @@ -1368,6 +1393,7 @@ export function isValidExtensionCursor(value: string): boolean { function parseDeveloper(row: DeveloperRow): PublicDeveloper { return { id: row.developerId, + identifier: row.developerIdentifier, type: row.developerType as "user" | "organization", name: row.developerName, URL: row.developerUrl ?? undefined, @@ -1426,7 +1452,7 @@ interface ListContentFields { function listContent( row: ListContentFields, card: true -): Omit; +): Omit; function listContent( row: ListContentFields, card: false @@ -1435,7 +1461,7 @@ function listContent( row: ListContentFields, card: boolean ): - | Omit + | Omit | Omit { const metadata = parseManifestMetadata(row.manifest); return { @@ -1469,7 +1495,7 @@ function listContent( // Shared keyset-pagination tail: each listing over-fetches one row to detect // hasMore, then encodes the next cursor from the final returned row. -function keysetPage( +function keysetPage( rows: T[], limit: number ): { pageRows: T[]; hasMore: boolean; nextCursor: string | null } { @@ -1482,7 +1508,7 @@ function keysetPage( nextCursor: hasMore && last ? extensionCursor.encode({ - normalizedId: last.id.toLowerCase(), + identifier: last.identifier.toLowerCase(), id: last.id }) : null @@ -1492,6 +1518,7 @@ function keysetPage( function parseListRow(row: PublishedListRow): ExtensionListItem { return { id: row.id, + identifier: row.identifier, ...listContent(row, true), developer: parseDeveloper(row) }; @@ -1502,6 +1529,7 @@ function parseListRow(row: PublishedListRow): ExtensionListItem { function parseRow(row: PublishedRow): Extension { return { id: row.id, + identifier: row.identifier, developer: parseDeveloper(row), ...listContent(row, false), homepage_url: row.website || undefined, @@ -1514,6 +1542,7 @@ function parseRow(row: PublishedRow): Extension { function parseOwnedListRow(row: OwnedListRow): OwnedExtensionListItem { return { id: row.id, + identifier: row.identifier, developer: parseDeveloper(row), published: row.publishedAt ? listContent(row, true) : null, pending_revision: diff --git a/src/services/extensions/v2/db/migrations/0032_directory_identity.sql b/src/services/extensions/v2/db/migrations/0032_directory_identity.sql new file mode 100644 index 00000000..31bed042 --- /dev/null +++ b/src/services/extensions/v2/db/migrations/0032_directory_identity.sql @@ -0,0 +1,688 @@ +-- Keep directory identities independent from public identifiers. +-- Rebuild children before parents so D1 can enforce foreign keys throughout. +CREATE TABLE _developers_ids AS SELECT id AS identifier, + substr(timestamp,1,8)||'-'||substr(timestamp,9,4)||'-7'||substr(entropy,1,3)||'-'||substr('89ab',1+(abs(random() % 4)),1)||substr(entropy,4,3)||'-'||substr(entropy,7,12) AS id + FROM (SELECT id,printf('%012x',CAST(unixepoch(created_at)*1000 AS INTEGER)) AS timestamp,lower(hex(randomblob(9))) AS entropy FROM developers); +--> statement-breakpoint +CREATE TABLE _extensions_ids AS SELECT id AS identifier, + substr(timestamp,1,8)||'-'||substr(timestamp,9,4)||'-7'||substr(entropy,1,3)||'-'||substr('89ab',1+(abs(random() % 4)),1)||substr(entropy,4,3)||'-'||substr(entropy,7,12) AS id + FROM (SELECT id,printf('%012x',CAST(unixepoch(created_at)*1000 AS INTEGER)) AS timestamp,lower(hex(randomblob(9))) AS entropy FROM extensions); +--> statement-breakpoint +CREATE TABLE _developers_backup AS SELECT rowid AS original_rowid,* FROM developers; +--> statement-breakpoint +CREATE TABLE _extensions_backup AS SELECT rowid AS original_rowid,* FROM extensions; +--> statement-breakpoint +CREATE TABLE _developer_transfers_backup AS SELECT rowid AS original_rowid,* FROM developer_transfers; +--> statement-breakpoint +CREATE TABLE _developer_claims_backup AS SELECT rowid AS original_rowid,* FROM developer_claims; +--> statement-breakpoint +CREATE TABLE _developer_history_backup AS SELECT rowid AS original_rowid,* FROM developer_history; +--> statement-breakpoint +CREATE TABLE _extension_write_events_backup AS SELECT rowid AS original_rowid,* FROM extension_write_events; +--> statement-breakpoint +CREATE TABLE _m2m_publisher_clients_backup AS SELECT rowid AS original_rowid,* FROM m2m_publisher_clients; +--> statement-breakpoint +CREATE TABLE _extension_revisions_backup AS SELECT rowid AS original_rowid,* FROM extension_revisions; +--> statement-breakpoint +DROP TRIGGER trg_developer_history_no_update; +--> statement-breakpoint +DROP TRIGGER trg_developer_history_no_delete; +--> statement-breakpoint +DROP TRIGGER resource_extension_revisions_insert; +--> statement-breakpoint +DROP TRIGGER resource_extension_revisions_delete; +--> statement-breakpoint +DROP TRIGGER resource_extension_revisions_update; +--> statement-breakpoint +DROP TRIGGER resource_extension_revisions_identity_update; +--> statement-breakpoint +DROP TRIGGER extension_revision_admission; +--> statement-breakpoint +DROP TRIGGER extension_revision_write_event; +--> statement-breakpoint +DROP TRIGGER extension_revision_content_bound; +--> statement-breakpoint +DROP TRIGGER extension_revision_identity; +--> statement-breakpoint +DROP TRIGGER extension_resource_identity; +--> statement-breakpoint +DROP TRIGGER sync_extension_revisions_bytes_insert; +--> statement-breakpoint +DROP TRIGGER sync_extension_revisions_bytes_update; +--> statement-breakpoint +DROP TRIGGER sync_revision_summary_insert; +--> statement-breakpoint +DROP TRIGGER sync_revision_summary_update; +--> statement-breakpoint +DROP TRIGGER developers_generate_profile_instance; +--> statement-breakpoint +DROP TRIGGER resource_extensions_insert; +--> statement-breakpoint +DROP TRIGGER sync_extensions_bytes_insert; +--> statement-breakpoint +DROP TRIGGER sync_extensions_bytes_update; +--> statement-breakpoint +DROP TRIGGER resource_extensions_delete; +--> statement-breakpoint +DROP TRIGGER resource_extensions_update; +--> statement-breakpoint +DROP TRIGGER extensions_published_manifest_insert; +--> statement-breakpoint +DROP TRIGGER extensions_published_manifest_update; +--> statement-breakpoint +DROP TRIGGER publish_package_artifacts; +--> statement-breakpoint +DROP TABLE extension_revisions; +--> statement-breakpoint +DROP TABLE m2m_publisher_clients; +--> statement-breakpoint +DROP TABLE extension_write_events; +--> statement-breakpoint +DROP TABLE developer_history; +--> statement-breakpoint +DROP TABLE developer_claims; +--> statement-breakpoint +DROP TABLE developer_transfers; +--> statement-breakpoint +DROP TABLE extensions; +--> statement-breakpoint +DROP TABLE developers; +--> statement-breakpoint +CREATE TABLE "developers" ( + id TEXT PRIMARY KEY NOT NULL, + identifier text NOT NULL, + type TEXT NOT NULL, + name TEXT NOT NULL, + url TEXT +, owner_user_id TEXT REFERENCES users(id), approved_at TEXT, created_at TEXT NOT NULL DEFAULT '1970-01-01T00:00:00.000Z', updated_at TEXT NOT NULL DEFAULT '1970-01-01T00:00:00.000Z', avatar_url TEXT, contact_email TEXT, ownership_epoch INTEGER NOT NULL DEFAULT 1 CHECK (ownership_epoch >= 1), content_revision INTEGER NOT NULL DEFAULT 1 CHECK (content_revision >= 1), approved_revision INTEGER, approved_by TEXT, github_org_verified INTEGER CHECK (github_org_verified IN (0, 1)), github_verification_note TEXT, `github_verified_at` text, github_url_verified INTEGER CHECK (github_url_verified = 1), `url_check_cooldown_until` text, `profile_generation` text DEFAULT '' NOT NULL); +--> statement-breakpoint +CREATE TABLE "extensions" ( + `id` text PRIMARY KEY NOT NULL, + identifier text NOT NULL, + `developer_id` text NOT NULL, + `published_at` text, + `published_revision_id` text, + `type` text, + `name` text, + `description` text, + `releases` text, + `website` text, + `license` text, + `icon_url` text, + `readme` text, + `source` text, + `version` text, + `download_url` text, + `created_at` text DEFAULT CURRENT_TIMESTAMP NOT NULL, + `updated_at` text DEFAULT CURRENT_TIMESTAMP NOT NULL, `delisted_at` text, `delist_reason` text, created_by TEXT, published_bytes INTEGER NOT NULL DEFAULT 0, manifest TEXT, + FOREIGN KEY (`developer_id`) REFERENCES `developers`(`id`) ON UPDATE no action ON DELETE no action, + CONSTRAINT "extensions_published_content_check" CHECK("extensions"."published_at" IS NULL OR ( + "extensions"."type" IS NOT NULL AND "extensions"."name" IS NOT NULL AND + "extensions"."description" IS NOT NULL AND "extensions"."releases" IS NOT NULL AND + "extensions"."website" IS NOT NULL AND "extensions"."license" IS NOT NULL AND + "extensions"."readme" IS NOT NULL AND "extensions"."source" IS NOT NULL AND + "extensions"."version" IS NOT NULL AND "extensions"."download_url" IS NOT NULL + )) +); +--> statement-breakpoint +CREATE TABLE "developer_transfers" ( + id TEXT PRIMARY KEY NOT NULL, + developer_id TEXT NOT NULL REFERENCES "developers"(id), + token_hash TEXT NOT NULL, + created_by TEXT NOT NULL REFERENCES users(id), + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + expires_at TEXT NOT NULL, + accepted_by TEXT REFERENCES users(id), + accepted_at TEXT, + revoked_at TEXT +); +--> statement-breakpoint +CREATE TABLE "developer_claims" ( + id TEXT PRIMARY KEY NOT NULL, + developer_id TEXT NOT NULL REFERENCES "developers"(id), + claimant_id TEXT NOT NULL REFERENCES users(id), + status TEXT NOT NULL DEFAULT 'pending' CHECK (status IN ('pending', 'approved', 'rejected')), + note TEXT, + review_note TEXT, + reviewer_id TEXT REFERENCES users(id), + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + reviewed_at TEXT +, github_org_verified INTEGER CHECK (github_org_verified IN (0, 1)), github_verification_note TEXT); +--> statement-breakpoint +CREATE TABLE "developer_history" ( + id TEXT PRIMARY KEY NOT NULL, + developer_id TEXT NOT NULL, + type TEXT NOT NULL, + name TEXT NOT NULL, + url TEXT, + changed_by TEXT NOT NULL REFERENCES users(id), + changed_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP +); +--> statement-breakpoint +CREATE TABLE extension_write_events ( + id TEXT PRIMARY KEY NOT NULL, account_id TEXT NOT NULL, + developer_id TEXT NOT NULL, occurred_at INTEGER NOT NULL +); +--> statement-breakpoint +CREATE TABLE `m2m_publisher_clients` ( + `client_id` text PRIMARY KEY NOT NULL, + `developer_id` text NOT NULL, + `can_publish` integer DEFAULT 0 NOT NULL, + `ownership_epoch` integer DEFAULT 1 NOT NULL, + `created_at` text DEFAULT CURRENT_TIMESTAMP NOT NULL, + `revoked_at` text, + FOREIGN KEY (`developer_id`) REFERENCES `developers`(`id`) ON UPDATE no action ON DELETE no action, + CONSTRAINT "m2m_publisher_clients_can_publish_check" CHECK("m2m_publisher_clients"."can_publish" IN (0, 1)), + CONSTRAINT "m2m_publisher_clients_ownership_epoch_check" CHECK("m2m_publisher_clients"."ownership_epoch" >= 1) +); +--> statement-breakpoint +CREATE TABLE `extension_revisions` ( + `id` text PRIMARY KEY NOT NULL, + `extension_id` text NOT NULL, + `developer_id` text NOT NULL, + `submitted_by` text NOT NULL, + `status` text DEFAULT 'pending' NOT NULL, + `content` text NOT NULL, + `reviewer_id` text, + `review_note` text, + `created_at` text DEFAULT CURRENT_TIMESTAMP NOT NULL, + `reviewed_at` text, + `ownership_epoch` integer DEFAULT 1 NOT NULL, content_bytes INTEGER NOT NULL DEFAULT 0, content_hash TEXT, compacted_at TEXT, summary_name TEXT, summary_version TEXT, summary_description TEXT, content_readable INTEGER NOT NULL DEFAULT 0, + FOREIGN KEY (`extension_id`) REFERENCES `extensions`(`id`) ON UPDATE no action ON DELETE cascade, + FOREIGN KEY (`submitted_by`) REFERENCES `users`(`id`) ON UPDATE no action ON DELETE no action, + FOREIGN KEY (`reviewer_id`) REFERENCES `users`(`id`) ON UPDATE no action ON DELETE no action, + CONSTRAINT "extension_revisions_status_check" CHECK("extension_revisions"."status" IN ('pending', 'approved', 'rejected')), + CONSTRAINT "extension_revisions_ownership_epoch_check" CHECK("extension_revisions"."ownership_epoch" >= 1) +); +--> statement-breakpoint +INSERT INTO developers (id,type,name,url,owner_user_id,approved_at,created_at,updated_at,avatar_url,contact_email,ownership_epoch,content_revision,approved_revision,approved_by,github_org_verified,github_verification_note,github_verified_at,github_url_verified,url_check_cooldown_until,profile_generation,identifier) + SELECT (SELECT id FROM _developers_ids WHERE identifier=b.id),b.type,b.name,b.url,b.owner_user_id,b.approved_at,b.created_at,b.updated_at,b.avatar_url,b.contact_email,b.ownership_epoch,b.content_revision,b.approved_revision,b.approved_by,b.github_org_verified,b.github_verification_note,b.github_verified_at,b.github_url_verified,b.url_check_cooldown_until,b.profile_generation,b.id + FROM _developers_backup b ORDER BY original_rowid; +--> statement-breakpoint +INSERT INTO extensions (id,developer_id,published_at,published_revision_id,type,name,description,releases,website,license,icon_url,readme,source,version,download_url,created_at,updated_at,delisted_at,delist_reason,created_by,published_bytes,manifest,identifier) + SELECT (SELECT id FROM _extensions_ids WHERE identifier=b.id),coalesce((SELECT id FROM _developers_ids WHERE identifier=b.developer_id),b.developer_id),b.published_at,b.published_revision_id,b.type,b.name,b.description,b.releases,b.website,b.license,b.icon_url,b.readme,b.source,b.version,b.download_url,b.created_at,b.updated_at,b.delisted_at,b.delist_reason,b.created_by,b.published_bytes,b.manifest,b.id + FROM _extensions_backup b ORDER BY original_rowid; +--> statement-breakpoint +INSERT INTO developer_transfers (id,developer_id,token_hash,created_by,created_at,expires_at,accepted_by,accepted_at,revoked_at) + SELECT b.id,coalesce((SELECT id FROM _developers_ids WHERE identifier=b.developer_id),b.developer_id),b.token_hash,b.created_by,b.created_at,b.expires_at,b.accepted_by,b.accepted_at,b.revoked_at + FROM _developer_transfers_backup b ORDER BY original_rowid; +--> statement-breakpoint +INSERT INTO developer_claims (id,developer_id,claimant_id,status,note,review_note,reviewer_id,created_at,reviewed_at,github_org_verified,github_verification_note) + SELECT b.id,coalesce((SELECT id FROM _developers_ids WHERE identifier=b.developer_id),b.developer_id),b.claimant_id,b.status,b.note,b.review_note,b.reviewer_id,b.created_at,b.reviewed_at,b.github_org_verified,b.github_verification_note + FROM _developer_claims_backup b ORDER BY original_rowid; +--> statement-breakpoint +INSERT INTO developer_history (id,developer_id,type,name,url,changed_by,changed_at) + SELECT b.id,coalesce((SELECT id FROM _developers_ids WHERE identifier=b.developer_id),b.developer_id),b.type,b.name,b.url,b.changed_by,b.changed_at + FROM _developer_history_backup b ORDER BY original_rowid; +--> statement-breakpoint +INSERT INTO extension_write_events (id,account_id,developer_id,occurred_at) + SELECT b.id,b.account_id,coalesce((SELECT id FROM _developers_ids WHERE identifier=b.developer_id),b.developer_id),b.occurred_at + FROM _extension_write_events_backup b ORDER BY original_rowid; +--> statement-breakpoint +INSERT INTO m2m_publisher_clients (client_id,developer_id,can_publish,ownership_epoch,created_at,revoked_at) + SELECT b.client_id,coalesce((SELECT id FROM _developers_ids WHERE identifier=b.developer_id),b.developer_id),b.can_publish,b.ownership_epoch,b.created_at,b.revoked_at + FROM _m2m_publisher_clients_backup b ORDER BY original_rowid; +--> statement-breakpoint +INSERT INTO extension_revisions (id,extension_id,developer_id,submitted_by,status,content,reviewer_id,review_note,created_at,reviewed_at,ownership_epoch,content_bytes,content_hash,compacted_at,summary_name,summary_version,summary_description,content_readable) + SELECT b.id,(SELECT id FROM _extensions_ids WHERE identifier=b.extension_id),coalesce((SELECT id FROM _developers_ids WHERE identifier=b.developer_id),b.developer_id),b.submitted_by,b.status,b.content,b.reviewer_id,b.review_note,b.created_at,b.reviewed_at,b.ownership_epoch,b.content_bytes,b.content_hash,b.compacted_at,b.summary_name,b.summary_version,b.summary_description,b.content_readable + FROM _extension_revisions_backup b ORDER BY original_rowid; +--> statement-breakpoint +CREATE UNIQUE INDEX idx_developers_owner_unique ON developers(owner_user_id); +--> statement-breakpoint +CREATE INDEX idx_developers_approved ON developers(approved_at); +--> statement-breakpoint +CREATE UNIQUE INDEX idx_developer_transfers_token ON developer_transfers(token_hash); +--> statement-breakpoint +CREATE UNIQUE INDEX idx_developer_transfers_pending + ON developer_transfers(developer_id) + WHERE accepted_at IS NULL AND revoked_at IS NULL; +--> statement-breakpoint +CREATE INDEX idx_developer_claims_developer ON developer_claims(developer_id); +--> statement-breakpoint +CREATE INDEX idx_developer_claims_claimant ON developer_claims(claimant_id); +--> statement-breakpoint +CREATE UNIQUE INDEX idx_developer_claims_pending_unique + ON developer_claims(developer_id, claimant_id) + WHERE status = 'pending'; +--> statement-breakpoint +CREATE INDEX idx_developer_claims_pending_queue + ON developer_claims(created_at) + WHERE status = 'pending'; +--> statement-breakpoint +CREATE INDEX idx_developer_history_developer_changed_at + ON developer_history(developer_id, changed_at); +--> statement-breakpoint +CREATE UNIQUE INDEX `idx_extensions_identifier_nocase` ON `extensions` (lower("identifier")); +--> statement-breakpoint +CREATE INDEX `idx_extensions_developer_order` ON `extensions` (`developer_id`,lower("identifier"),`id`); +--> statement-breakpoint +CREATE INDEX `idx_extension_revisions_submitted_by` ON `extension_revisions` (`submitted_by`); +--> statement-breakpoint +CREATE INDEX `idx_extension_revisions_developer` ON `extension_revisions` (`developer_id`); +--> statement-breakpoint +CREATE UNIQUE INDEX `idx_extension_revisions_pending` ON `extension_revisions` (`extension_id`) WHERE "extension_revisions"."status" = 'pending'; +--> statement-breakpoint +CREATE INDEX `idx_extension_revisions_extension_page` ON `extension_revisions` (`extension_id`,"created_at" desc,"id" desc); +--> statement-breakpoint +CREATE INDEX `idx_extension_revisions_submitter_page` ON `extension_revisions` (`submitted_by`,"created_at" desc,"id" desc); +--> statement-breakpoint +CREATE INDEX `idx_extension_revisions_queue_page` ON `extension_revisions` (`status`,`created_at`,`id`); +--> statement-breakpoint +CREATE INDEX `idx_extension_revisions_reviewed` ON `extension_revisions` (`extension_id`,`reviewed_at`) WHERE "extension_revisions"."status" IN ('approved', 'rejected'); +--> statement-breakpoint +CREATE INDEX `idx_extension_revisions_submitter_pending` ON `extension_revisions` (`submitted_by`) WHERE "extension_revisions"."status" = 'pending'; +--> statement-breakpoint +CREATE INDEX `idx_developer_history_account_changed_at` ON `developer_history` (`changed_by`,`changed_at`); +--> statement-breakpoint +CREATE INDEX idx_extension_write_events_account ON extension_write_events(account_id,occurred_at); +--> statement-breakpoint +CREATE INDEX idx_extension_write_events_developer ON extension_write_events(developer_id,occurred_at); +--> statement-breakpoint +CREATE INDEX idx_extension_write_events_time ON extension_write_events(occurred_at); +--> statement-breakpoint +CREATE INDEX idx_extensions_published_revision ON extensions(published_revision_id); +--> statement-breakpoint +CREATE INDEX idx_extension_revisions_retention ON extension_revisions(compacted_at,reviewed_at,id,status); +--> statement-breakpoint +CREATE INDEX `idx_m2m_publisher_clients_developer` ON `m2m_publisher_clients` (`developer_id`); +--> statement-breakpoint +CREATE INDEX idx_extensions_public_order ON extensions (lower(id),id) WHERE published_at IS NOT NULL AND delisted_at IS NULL; +--> statement-breakpoint +CREATE INDEX idx_extensions_type_public_order ON extensions (type,lower(id),id) WHERE published_at IS NOT NULL AND delisted_at IS NULL; +--> statement-breakpoint +CREATE UNIQUE INDEX idx_developers_identifier_nocase ON developers(lower(identifier)); +--> statement-breakpoint +CREATE TRIGGER trg_developer_history_no_update +BEFORE UPDATE ON developer_history +BEGIN + SELECT RAISE(ABORT, 'developer_history is append-only: rows cannot be updated'); +END; +--> statement-breakpoint +CREATE TRIGGER trg_developer_history_no_delete +BEFORE DELETE ON developer_history +BEGIN + SELECT RAISE(ABORT, 'developer_history is append-only: rows cannot be deleted'); +END; +--> statement-breakpoint +CREATE TRIGGER resource_extension_revisions_insert AFTER INSERT ON extension_revisions BEGIN + INSERT INTO extension_resource_usage(scope,subject,bytes,extensions,revisions) + SELECT scope, subject, length(CAST(NEW.content AS BLOB)), 0, 1 + FROM ( + SELECT 'global' AS scope, 'all' AS subject + UNION ALL SELECT 'account', NEW.submitted_by + UNION ALL SELECT 'developer', NEW.developer_id + ) WHERE true + ON CONFLICT(scope,subject) DO UPDATE SET + bytes=bytes+excluded.bytes, + extensions=extensions+excluded.extensions, + revisions=revisions+excluded.revisions; + SELECT CASE WHEN EXISTS ( + SELECT 1 FROM extension_resource_usage + WHERE ((scope='account' AND subject=NEW.submitted_by) + OR (scope='developer' AND subject=NEW.developer_id)) + AND (bytes>26214400 OR extensions>100 OR revisions>1000) + ) THEN RAISE(ABORT,'extension_resource_quota') END; +END; +--> statement-breakpoint +CREATE TRIGGER resource_extension_revisions_delete AFTER DELETE ON extension_revisions BEGIN + UPDATE extension_resource_usage + SET bytes=bytes-(length(CAST(OLD.content AS BLOB))), revisions=revisions-1 + WHERE (scope='global' AND subject='all') + OR (scope='account' AND subject=OLD.submitted_by) + OR (scope='developer' AND subject=OLD.developer_id); +END; +--> statement-breakpoint +CREATE TRIGGER resource_extension_revisions_update AFTER UPDATE OF content ON extension_revisions +WHEN NEW.submitted_by=OLD.submitted_by AND NEW.developer_id=OLD.developer_id BEGIN + UPDATE extension_resource_usage + SET bytes = bytes + length(CAST(NEW.content AS BLOB)) - length(CAST(OLD.content AS BLOB)) + WHERE (scope='global' AND subject='all') + OR (scope='account' AND subject=NEW.submitted_by) + OR (scope='developer' AND subject=NEW.developer_id); + SELECT CASE WHEN length(CAST(NEW.content AS BLOB)) > length(CAST(OLD.content AS BLOB)) AND EXISTS ( + SELECT 1 FROM extension_resource_usage + WHERE ((scope='account' AND subject=NEW.submitted_by) + OR (scope='developer' AND subject=NEW.developer_id)) + AND (bytes>26214400 OR extensions>100 OR revisions>1000) + ) + THEN RAISE(ABORT,'extension_resource_quota') END; +END; +--> statement-breakpoint +CREATE TRIGGER resource_extension_revisions_identity_update AFTER UPDATE OF submitted_by,developer_id ON extension_revisions +WHEN NEW.submitted_by IS NOT OLD.submitted_by OR NEW.developer_id IS NOT OLD.developer_id BEGIN +UPDATE extension_resource_usage SET bytes=bytes+((length(CAST(OLD.content AS BLOB))) * -1), revisions=revisions+(-1) + WHERE scope='global' AND subject='all'; +UPDATE extension_resource_usage SET bytes=bytes+((length(CAST(OLD.content AS BLOB))) * -1), revisions=revisions+(-1) + WHERE scope='account' AND subject=OLD.submitted_by; +UPDATE extension_resource_usage SET bytes=bytes+((length(CAST(OLD.content AS BLOB))) * -1), revisions=revisions+(-1) + WHERE scope='developer' AND subject=OLD.developer_id; +INSERT INTO extension_resource_usage(scope,subject,bytes,extensions,revisions) VALUES ('global','all',(length(CAST(NEW.content AS BLOB))),0,1) + ON CONFLICT(scope,subject) DO UPDATE SET + bytes=bytes+excluded.bytes,extensions=extensions+excluded.extensions,revisions=revisions+excluded.revisions; +INSERT INTO extension_resource_usage(scope,subject,bytes,extensions,revisions) VALUES ('account',NEW.submitted_by,(length(CAST(NEW.content AS BLOB))),0,1) + ON CONFLICT(scope,subject) DO UPDATE SET + bytes=bytes+excluded.bytes,extensions=extensions+excluded.extensions,revisions=revisions+excluded.revisions; +SELECT CASE WHEN ((length(CAST(NEW.content AS BLOB))) > (length(CAST(OLD.content AS BLOB)))) AND EXISTS (SELECT 1 FROM extension_resource_usage + WHERE scope='account' AND subject=NEW.submitted_by AND (bytes>26214400 OR extensions>100 OR revisions>1000)) + THEN RAISE(ABORT,'extension_resource_quota') END; +INSERT INTO extension_resource_usage(scope,subject,bytes,extensions,revisions) VALUES ('developer',NEW.developer_id,(length(CAST(NEW.content AS BLOB))),0,1) + ON CONFLICT(scope,subject) DO UPDATE SET + bytes=bytes+excluded.bytes,extensions=extensions+excluded.extensions,revisions=revisions+excluded.revisions; +SELECT CASE WHEN ((length(CAST(NEW.content AS BLOB))) > (length(CAST(OLD.content AS BLOB)))) AND EXISTS (SELECT 1 FROM extension_resource_usage + WHERE scope='developer' AND subject=NEW.developer_id AND (bytes>26214400 OR extensions>100 OR revisions>1000)) + THEN RAISE(ABORT,'extension_resource_quota') END; +END; +--> statement-breakpoint +CREATE TRIGGER extension_revision_admission BEFORE INSERT ON extension_revisions BEGIN + -- Cleanup scales with accepted writes; idle cleanup still runs hourly. + DELETE FROM extension_write_events WHERE id IN ( + SELECT id FROM extension_write_events WHERE occurred_at <= unixepoch()-86400 + ORDER BY occurred_at LIMIT 50 + ); + SELECT CASE WHEN length(CAST(NEW.content AS BLOB)) > 262144 + + THEN RAISE(ABORT,'extension_content_size') END; + SELECT CASE WHEN (SELECT COUNT(*) FROM extension_write_events WHERE account_id=NEW.submitted_by AND occurred_at>unixepoch()-60)>=5 + OR (SELECT COUNT(*) FROM extension_write_events WHERE developer_id=NEW.developer_id AND occurred_at>unixepoch()-60)>=5 + + THEN RAISE(ABORT,'extension_write_rate_minute') END; + SELECT CASE WHEN (SELECT COUNT(*) FROM extension_write_events WHERE account_id=NEW.submitted_by AND occurred_at>unixepoch()-86400)>=50 + OR (SELECT COUNT(*) FROM extension_write_events WHERE developer_id=NEW.developer_id AND occurred_at>unixepoch()-86400)>=50 + + THEN RAISE(ABORT,'extension_write_rate_day') END; +END; +--> statement-breakpoint +CREATE TRIGGER extension_revision_write_event AFTER INSERT ON extension_revisions BEGIN + INSERT INTO extension_write_events(id,account_id,developer_id,occurred_at) + VALUES (NEW.id,NEW.submitted_by,NEW.developer_id,unixepoch(NEW.created_at)); +END; +--> statement-breakpoint +CREATE TRIGGER extension_revision_content_bound BEFORE UPDATE OF content ON extension_revisions +WHEN length(CAST(NEW.content AS BLOB)) > 262144 AND length(CAST(NEW.content AS BLOB)) > length(CAST(OLD.content AS BLOB)) +BEGIN SELECT RAISE(ABORT,'extension_content_size'); END; +--> statement-breakpoint +CREATE TRIGGER extension_revision_identity BEFORE UPDATE OF id,extension_id,developer_id,submitted_by ON extension_revisions +WHEN NEW.id IS NOT OLD.id OR NEW.extension_id IS NOT OLD.extension_id + OR NEW.developer_id IS NOT OLD.developer_id OR NEW.submitted_by IS NOT OLD.submitted_by +BEGIN SELECT RAISE(ABORT,'extension_resource_identity'); END; +--> statement-breakpoint +CREATE TRIGGER extension_resource_identity BEFORE UPDATE OF created_by,developer_id ON extensions +WHEN NEW.created_by IS NOT OLD.created_by OR NEW.developer_id IS NOT OLD.developer_id +BEGIN SELECT RAISE(ABORT,'extension_resource_identity'); END; +--> statement-breakpoint +CREATE TRIGGER sync_extension_revisions_bytes_insert AFTER INSERT ON extension_revisions +WHEN NEW.content_bytes != (length(CAST(NEW.content AS BLOB))) BEGIN + UPDATE extension_revisions SET content_bytes=(length(CAST(NEW.content AS BLOB))) + WHERE id=NEW.id; +END; +--> statement-breakpoint +CREATE TRIGGER sync_extension_revisions_bytes_update AFTER UPDATE OF content,content_bytes ON extension_revisions +WHEN NEW.content_bytes != (length(CAST(NEW.content AS BLOB))) BEGIN + UPDATE extension_revisions SET content_bytes=(length(CAST(NEW.content AS BLOB))) + WHERE id=NEW.id; +END; +--> statement-breakpoint +CREATE TRIGGER sync_revision_summary_insert AFTER INSERT ON extension_revisions +WHEN NEW.compacted_at IS NULL BEGIN + UPDATE extension_revisions SET content_readable = CASE WHEN length(CAST(NEW.content AS BLOB)) <= 262144 AND json_valid(NEW.content) THEN + CASE WHEN json_type(NEW.content) != 'object' THEN 0 + WHEN json_type(NEW.content, '$.releases') IS NULL OR json_type(NEW.content, '$.releases') = 'null' THEN 1 + WHEN json_type(NEW.content, '$.releases') != 'array' THEN 0 + WHEN json_array_length(NEW.content, '$.releases') > 100 THEN 0 + ELSE NOT EXISTS (SELECT 1 FROM json_each(NEW.content, '$.releases') r + WHERE CASE WHEN r.type = 'object' THEN + json_type(r.value, '$.tag') IS NOT 'text' OR length(json_extract(r.value, '$.tag')) > 100 + ELSE 1 END) + END ELSE 0 END, + summary_name = CASE WHEN length(CAST(NEW.content AS BLOB)) <= 262144 AND json_valid(NEW.content) + THEN CASE WHEN json_type(NEW.content,'$.name')='text' THEN substr(json_extract(NEW.content,'$.name'),1,120) ELSE NULL END + ELSE NULL END, + summary_version = CASE WHEN length(CAST(NEW.content AS BLOB)) <= 262144 AND json_valid(NEW.content) + THEN CASE WHEN json_type(NEW.content,'$.version')='text' THEN substr(json_extract(NEW.content,'$.version'),1,100) ELSE NULL END + ELSE NULL END, + summary_description = CASE WHEN length(CAST(NEW.content AS BLOB)) <= 262144 AND json_valid(NEW.content) + THEN CASE WHEN json_type(NEW.content,'$.description')='text' THEN substr(json_extract(NEW.content,'$.description'),1,4000) ELSE NULL END + ELSE NULL END + WHERE id=NEW.id; +END; +--> statement-breakpoint +CREATE TRIGGER sync_revision_summary_update AFTER UPDATE OF content ON extension_revisions +WHEN NEW.compacted_at IS NULL BEGIN + UPDATE extension_revisions SET content_readable = CASE WHEN length(CAST(NEW.content AS BLOB)) <= 262144 AND json_valid(NEW.content) THEN + CASE WHEN json_type(NEW.content) != 'object' THEN 0 + WHEN json_type(NEW.content, '$.releases') IS NULL OR json_type(NEW.content, '$.releases') = 'null' THEN 1 + WHEN json_type(NEW.content, '$.releases') != 'array' THEN 0 + WHEN json_array_length(NEW.content, '$.releases') > 100 THEN 0 + ELSE NOT EXISTS (SELECT 1 FROM json_each(NEW.content, '$.releases') r + WHERE CASE WHEN r.type = 'object' THEN + json_type(r.value, '$.tag') IS NOT 'text' OR length(json_extract(r.value, '$.tag')) > 100 + ELSE 1 END) + END ELSE 0 END, + summary_name = CASE WHEN length(CAST(NEW.content AS BLOB)) <= 262144 AND json_valid(NEW.content) + THEN CASE WHEN json_type(NEW.content,'$.name')='text' THEN substr(json_extract(NEW.content,'$.name'),1,120) ELSE NULL END + ELSE NULL END, + summary_version = CASE WHEN length(CAST(NEW.content AS BLOB)) <= 262144 AND json_valid(NEW.content) + THEN CASE WHEN json_type(NEW.content,'$.version')='text' THEN substr(json_extract(NEW.content,'$.version'),1,100) ELSE NULL END + ELSE NULL END, + summary_description = CASE WHEN length(CAST(NEW.content AS BLOB)) <= 262144 AND json_valid(NEW.content) + THEN CASE WHEN json_type(NEW.content,'$.description')='text' THEN substr(json_extract(NEW.content,'$.description'),1,4000) ELSE NULL END + ELSE NULL END + WHERE id=NEW.id; +END; +--> statement-breakpoint +CREATE TRIGGER developers_generate_profile_instance +AFTER INSERT ON developers +BEGIN + UPDATE developers SET profile_generation = lower(hex(randomblob(16))) + WHERE id = NEW.id; +END; +--> statement-breakpoint +CREATE TRIGGER resource_extensions_insert AFTER INSERT ON extensions BEGIN + INSERT INTO extension_resource_usage(scope,subject,bytes,extensions,revisions) + SELECT scope, subject, COALESCE(length(CAST(NEW.manifest AS BLOB)),0) + + COALESCE(length(CAST(NEW.type AS BLOB)),0) + + COALESCE(length(CAST(NEW.name AS BLOB)),0) + + COALESCE(length(CAST(NEW.description AS BLOB)),0) + + COALESCE(length(CAST(NEW.releases AS BLOB)),0) + + COALESCE(length(CAST(NEW.website AS BLOB)),0) + + COALESCE(length(CAST(NEW.license AS BLOB)),0) + + COALESCE(length(CAST(NEW.icon_url AS BLOB)),0) + + COALESCE(length(CAST(NEW.readme AS BLOB)),0) + + COALESCE(length(CAST(NEW.source AS BLOB)),0) + + COALESCE(length(CAST(NEW.version AS BLOB)),0) + + COALESCE(length(CAST(NEW.download_url AS BLOB)),0), 1, 0 + FROM ( + SELECT 'global' AS scope, 'all' AS subject + UNION ALL SELECT 'account', COALESCE(NEW.created_by,'legacy') + UNION ALL SELECT 'developer', NEW.developer_id + ) WHERE true + ON CONFLICT(scope,subject) DO UPDATE SET + bytes=bytes+excluded.bytes, + extensions=extensions+excluded.extensions, + revisions=revisions+excluded.revisions; + SELECT CASE WHEN EXISTS ( + SELECT 1 FROM extension_resource_usage + WHERE ((scope='account' AND subject=COALESCE(NEW.created_by,'legacy')) + OR (scope='developer' AND subject=NEW.developer_id)) + AND (bytes>26214400 OR extensions>100 OR revisions>1000) + ) THEN RAISE(ABORT,'extension_resource_quota') END; +END; +--> statement-breakpoint +CREATE TRIGGER sync_extensions_bytes_insert AFTER INSERT ON extensions +WHEN NEW.published_bytes != (COALESCE(length(CAST(NEW.manifest AS BLOB)),0) + + COALESCE(length(CAST(NEW.type AS BLOB)),0) + + COALESCE(length(CAST(NEW.name AS BLOB)),0) + + COALESCE(length(CAST(NEW.description AS BLOB)),0) + + COALESCE(length(CAST(NEW.releases AS BLOB)),0) + + COALESCE(length(CAST(NEW.website AS BLOB)),0) + + COALESCE(length(CAST(NEW.license AS BLOB)),0) + + COALESCE(length(CAST(NEW.icon_url AS BLOB)),0) + + COALESCE(length(CAST(NEW.readme AS BLOB)),0) + + COALESCE(length(CAST(NEW.source AS BLOB)),0) + + COALESCE(length(CAST(NEW.version AS BLOB)),0) + + COALESCE(length(CAST(NEW.download_url AS BLOB)),0)) BEGIN + UPDATE extensions SET published_bytes=(COALESCE(length(CAST(NEW.manifest AS BLOB)),0) + + COALESCE(length(CAST(NEW.type AS BLOB)),0) + + COALESCE(length(CAST(NEW.name AS BLOB)),0) + + COALESCE(length(CAST(NEW.description AS BLOB)),0) + + COALESCE(length(CAST(NEW.releases AS BLOB)),0) + + COALESCE(length(CAST(NEW.website AS BLOB)),0) + + COALESCE(length(CAST(NEW.license AS BLOB)),0) + + COALESCE(length(CAST(NEW.icon_url AS BLOB)),0) + + COALESCE(length(CAST(NEW.readme AS BLOB)),0) + + COALESCE(length(CAST(NEW.source AS BLOB)),0) + + COALESCE(length(CAST(NEW.version AS BLOB)),0) + + COALESCE(length(CAST(NEW.download_url AS BLOB)),0)) + WHERE id=NEW.id; +END; +--> statement-breakpoint +CREATE TRIGGER sync_extensions_bytes_update AFTER UPDATE OF type,name,description,releases,website,license,icon_url,readme,source,version,download_url,manifest,published_bytes ON extensions +WHEN NEW.published_bytes != (COALESCE(length(CAST(NEW.manifest AS BLOB)),0) + + COALESCE(length(CAST(NEW.type AS BLOB)),0) + + COALESCE(length(CAST(NEW.name AS BLOB)),0) + + COALESCE(length(CAST(NEW.description AS BLOB)),0) + + COALESCE(length(CAST(NEW.releases AS BLOB)),0) + + COALESCE(length(CAST(NEW.website AS BLOB)),0) + + COALESCE(length(CAST(NEW.license AS BLOB)),0) + + COALESCE(length(CAST(NEW.icon_url AS BLOB)),0) + + COALESCE(length(CAST(NEW.readme AS BLOB)),0) + + COALESCE(length(CAST(NEW.source AS BLOB)),0) + + COALESCE(length(CAST(NEW.version AS BLOB)),0) + + COALESCE(length(CAST(NEW.download_url AS BLOB)),0)) BEGIN + UPDATE extensions SET published_bytes=(COALESCE(length(CAST(NEW.manifest AS BLOB)),0) + + COALESCE(length(CAST(NEW.type AS BLOB)),0) + + COALESCE(length(CAST(NEW.name AS BLOB)),0) + + COALESCE(length(CAST(NEW.description AS BLOB)),0) + + COALESCE(length(CAST(NEW.releases AS BLOB)),0) + + COALESCE(length(CAST(NEW.website AS BLOB)),0) + + COALESCE(length(CAST(NEW.license AS BLOB)),0) + + COALESCE(length(CAST(NEW.icon_url AS BLOB)),0) + + COALESCE(length(CAST(NEW.readme AS BLOB)),0) + + COALESCE(length(CAST(NEW.source AS BLOB)),0) + + COALESCE(length(CAST(NEW.version AS BLOB)),0) + + COALESCE(length(CAST(NEW.download_url AS BLOB)),0)) + WHERE id=NEW.id; +END; +--> statement-breakpoint +CREATE TRIGGER resource_extensions_delete AFTER DELETE ON extensions BEGIN + UPDATE extension_resource_usage + SET bytes=bytes-(COALESCE(length(CAST(OLD.manifest AS BLOB)),0) + COALESCE(length(CAST(OLD.type AS BLOB)),0) + + COALESCE(length(CAST(OLD.name AS BLOB)),0) + + COALESCE(length(CAST(OLD.description AS BLOB)),0) + + COALESCE(length(CAST(OLD.releases AS BLOB)),0) + + COALESCE(length(CAST(OLD.website AS BLOB)),0) + + COALESCE(length(CAST(OLD.license AS BLOB)),0) + + COALESCE(length(CAST(OLD.icon_url AS BLOB)),0) + + COALESCE(length(CAST(OLD.readme AS BLOB)),0) + + COALESCE(length(CAST(OLD.source AS BLOB)),0) + + COALESCE(length(CAST(OLD.version AS BLOB)),0) + + COALESCE(length(CAST(OLD.download_url AS BLOB)),0)), extensions=extensions-1 + WHERE (scope='global' AND subject='all') + OR (scope='account' AND subject=COALESCE(OLD.created_by,'legacy')) + OR (scope='developer' AND subject=OLD.developer_id); +END; +--> statement-breakpoint +CREATE TRIGGER resource_extensions_update AFTER UPDATE OF type,name,description,releases,website,license,icon_url,readme,source,version,download_url,manifest ON extensions BEGIN + UPDATE extension_resource_usage + SET bytes = bytes + (COALESCE(length(CAST(NEW.manifest AS BLOB)),0) + + COALESCE(length(CAST(NEW.type AS BLOB)),0) + + COALESCE(length(CAST(NEW.name AS BLOB)),0) + + COALESCE(length(CAST(NEW.description AS BLOB)),0) + + COALESCE(length(CAST(NEW.releases AS BLOB)),0) + + COALESCE(length(CAST(NEW.website AS BLOB)),0) + + COALESCE(length(CAST(NEW.license AS BLOB)),0) + + COALESCE(length(CAST(NEW.icon_url AS BLOB)),0) + + COALESCE(length(CAST(NEW.readme AS BLOB)),0) + + COALESCE(length(CAST(NEW.source AS BLOB)),0) + + COALESCE(length(CAST(NEW.version AS BLOB)),0) + + COALESCE(length(CAST(NEW.download_url AS BLOB)),0)) - (COALESCE(length(CAST(OLD.manifest AS BLOB)),0) + + COALESCE(length(CAST(OLD.type AS BLOB)),0) + + COALESCE(length(CAST(OLD.name AS BLOB)),0) + + COALESCE(length(CAST(OLD.description AS BLOB)),0) + + COALESCE(length(CAST(OLD.releases AS BLOB)),0) + + COALESCE(length(CAST(OLD.website AS BLOB)),0) + + COALESCE(length(CAST(OLD.license AS BLOB)),0) + + COALESCE(length(CAST(OLD.icon_url AS BLOB)),0) + + COALESCE(length(CAST(OLD.readme AS BLOB)),0) + + COALESCE(length(CAST(OLD.source AS BLOB)),0) + + COALESCE(length(CAST(OLD.version AS BLOB)),0) + + COALESCE(length(CAST(OLD.download_url AS BLOB)),0)) + WHERE (scope='global' AND subject='all') + OR (scope='account' AND subject=COALESCE(NEW.created_by,'legacy')) + OR (scope='developer' AND subject=NEW.developer_id); + SELECT CASE WHEN (COALESCE(length(CAST(NEW.manifest AS BLOB)),0) + + COALESCE(length(CAST(NEW.type AS BLOB)),0) + + COALESCE(length(CAST(NEW.name AS BLOB)),0) + + COALESCE(length(CAST(NEW.description AS BLOB)),0) + + COALESCE(length(CAST(NEW.releases AS BLOB)),0) + + COALESCE(length(CAST(NEW.website AS BLOB)),0) + + COALESCE(length(CAST(NEW.license AS BLOB)),0) + + COALESCE(length(CAST(NEW.icon_url AS BLOB)),0) + + COALESCE(length(CAST(NEW.readme AS BLOB)),0) + + COALESCE(length(CAST(NEW.source AS BLOB)),0) + + COALESCE(length(CAST(NEW.version AS BLOB)),0) + + COALESCE(length(CAST(NEW.download_url AS BLOB)),0)) > (COALESCE(length(CAST(OLD.manifest AS BLOB)),0) + + COALESCE(length(CAST(OLD.type AS BLOB)),0) + + COALESCE(length(CAST(OLD.name AS BLOB)),0) + + COALESCE(length(CAST(OLD.description AS BLOB)),0) + + COALESCE(length(CAST(OLD.releases AS BLOB)),0) + + COALESCE(length(CAST(OLD.website AS BLOB)),0) + + COALESCE(length(CAST(OLD.license AS BLOB)),0) + + COALESCE(length(CAST(OLD.icon_url AS BLOB)),0) + + COALESCE(length(CAST(OLD.readme AS BLOB)),0) + + COALESCE(length(CAST(OLD.source AS BLOB)),0) + + COALESCE(length(CAST(OLD.version AS BLOB)),0) + + COALESCE(length(CAST(OLD.download_url AS BLOB)),0)) AND EXISTS ( + SELECT 1 FROM extension_resource_usage + WHERE ((scope='account' AND subject=COALESCE(NEW.created_by,'legacy')) + OR (scope='developer' AND subject=NEW.developer_id)) + AND (bytes>26214400 OR extensions>100 OR revisions>1000) + ) + THEN RAISE(ABORT,'extension_resource_quota') END; +END; +--> statement-breakpoint +CREATE TRIGGER extensions_published_manifest_insert BEFORE INSERT ON extensions +WHEN NEW.published_at IS NOT NULL AND NEW.manifest IS NULL +BEGIN SELECT RAISE(ABORT,'extensions_published_manifest_required'); END; +--> statement-breakpoint +CREATE TRIGGER extensions_published_manifest_update BEFORE UPDATE OF published_at,manifest ON extensions +WHEN NEW.published_at IS NOT NULL AND NEW.manifest IS NULL +BEGIN SELECT RAISE(ABORT,'extensions_published_manifest_required'); END; +--> statement-breakpoint +CREATE TRIGGER publish_package_artifacts AFTER UPDATE OF published_revision_id ON extensions +WHEN NEW.published_revision_id IS NOT NULL AND NEW.published_revision_id IS NOT OLD.published_revision_id +BEGIN + UPDATE package_artifacts SET published_at=CURRENT_TIMESTAMP + WHERE published_at IS NULL AND id IN ( + SELECT json_extract(value,'$.artifact_id') FROM json_each(NEW.releases) + ) AND source_id IN ( + SELECT s.id FROM package_sources s JOIN developers d ON d.profile_generation=s.profile_generation + WHERE s.extension_id=lower(NEW.identifier) AND d.id=NEW.developer_id + ); +END; +--> statement-breakpoint +UPDATE extension_resource_usage SET subject=(SELECT id FROM _developers_ids WHERE identifier=subject) WHERE scope='developer' AND subject IN (SELECT identifier FROM _developers_ids); +--> statement-breakpoint +UPDATE package_jobs SET owner=json_set(owner,'$.id',(SELECT id FROM _developers_ids WHERE identifier=json_extract(owner,'$.id'))) WHERE json_extract(owner,'$.id') IN (SELECT identifier FROM _developers_ids); +--> statement-breakpoint +ALTER TABLE package_sources RENAME COLUMN extension_id TO extension_identifier; +--> statement-breakpoint +DROP TABLE _developers_backup; +--> statement-breakpoint +DROP TABLE _extensions_backup; +--> statement-breakpoint +DROP TABLE _developer_transfers_backup; +--> statement-breakpoint +DROP TABLE _developer_claims_backup; +--> statement-breakpoint +DROP TABLE _developer_history_backup; +--> statement-breakpoint +DROP TABLE _extension_write_events_backup; +--> statement-breakpoint +DROP TABLE _m2m_publisher_clients_backup; +--> statement-breakpoint +DROP TABLE _extension_revisions_backup; +--> statement-breakpoint +DROP TABLE _developers_ids; +--> statement-breakpoint +DROP TABLE _extensions_ids; diff --git a/src/services/extensions/v2/db/migrations/meta/0032_snapshot.json b/src/services/extensions/v2/db/migrations/meta/0032_snapshot.json new file mode 100644 index 00000000..9b205aaa --- /dev/null +++ b/src/services/extensions/v2/db/migrations/meta/0032_snapshot.json @@ -0,0 +1,1929 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "97b5f6c8-65fb-4691-9c27-e65e94ab3987", + "prevId": "67cf78d3-42a6-4777-a988-ebf2d8c4f9a9", + "tables": { + "claim_verification_budgets": { + "name": "claim_verification_budgets", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_claim_verification_budgets_expiry": { + "name": "idx_claim_verification_budgets_expiry", + "columns": ["expires_at"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "developer_claims": { + "name": "developer_claims", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "developer_id": { + "name": "developer_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "claimant_id": { + "name": "claimant_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "note": { + "name": "note", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "review_note": { + "name": "review_note", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reviewer_id": { + "name": "reviewer_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + }, + "reviewed_at": { + "name": "reviewed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "github_org_verified": { + "name": "github_org_verified", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "github_verification_note": { + "name": "github_verification_note", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "idx_developer_claims_developer": { + "name": "idx_developer_claims_developer", + "columns": ["developer_id"], + "isUnique": false + }, + "idx_developer_claims_claimant": { + "name": "idx_developer_claims_claimant", + "columns": ["claimant_id"], + "isUnique": false + }, + "idx_developer_claims_pending_unique": { + "name": "idx_developer_claims_pending_unique", + "columns": ["developer_id", "claimant_id"], + "isUnique": true, + "where": "\"developer_claims\".\"status\" = 'pending'" + }, + "idx_developer_claims_pending_queue": { + "name": "idx_developer_claims_pending_queue", + "columns": ["created_at"], + "isUnique": false, + "where": "\"developer_claims\".\"status\" = 'pending'" + } + }, + "foreignKeys": { + "developer_claims_developer_id_developers_id_fk": { + "name": "developer_claims_developer_id_developers_id_fk", + "tableFrom": "developer_claims", + "tableTo": "developers", + "columnsFrom": ["developer_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "developer_claims_claimant_id_users_id_fk": { + "name": "developer_claims_claimant_id_users_id_fk", + "tableFrom": "developer_claims", + "tableTo": "users", + "columnsFrom": ["claimant_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "developer_claims_reviewer_id_users_id_fk": { + "name": "developer_claims_reviewer_id_users_id_fk", + "tableFrom": "developer_claims", + "tableTo": "users", + "columnsFrom": ["reviewer_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": { + "developer_claims_status_check": { + "name": "developer_claims_status_check", + "value": "\"developer_claims\".\"status\" IN ('pending', 'approved', 'rejected')" + }, + "developer_claims_github_org_verified_check": { + "name": "developer_claims_github_org_verified_check", + "value": "\"developer_claims\".\"github_org_verified\" IN (0, 1)" + } + } + }, + "developer_history": { + "name": "developer_history", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "developer_id": { + "name": "developer_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "changed_by": { + "name": "changed_by", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "changed_at": { + "name": "changed_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + } + }, + "indexes": { + "idx_developer_history_developer_changed_at": { + "name": "idx_developer_history_developer_changed_at", + "columns": ["developer_id", "changed_at"], + "isUnique": false + }, + "idx_developer_history_account_changed_at": { + "name": "idx_developer_history_account_changed_at", + "columns": ["changed_by", "changed_at"], + "isUnique": false + } + }, + "foreignKeys": { + "developer_history_changed_by_users_id_fk": { + "name": "developer_history_changed_by_users_id_fk", + "tableFrom": "developer_history", + "tableTo": "users", + "columnsFrom": ["changed_by"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "developer_transfers": { + "name": "developer_transfers", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "developer_id": { + "name": "developer_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token_hash": { + "name": "token_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + }, + "expires_at": { + "name": "expires_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "accepted_by": { + "name": "accepted_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "accepted_at": { + "name": "accepted_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "revoked_at": { + "name": "revoked_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "idx_developer_transfers_token": { + "name": "idx_developer_transfers_token", + "columns": ["token_hash"], + "isUnique": true + }, + "idx_developer_transfers_pending": { + "name": "idx_developer_transfers_pending", + "columns": ["developer_id"], + "isUnique": true, + "where": "\"developer_transfers\".\"accepted_at\" IS NULL AND \"developer_transfers\".\"revoked_at\" IS NULL" + } + }, + "foreignKeys": { + "developer_transfers_developer_id_developers_id_fk": { + "name": "developer_transfers_developer_id_developers_id_fk", + "tableFrom": "developer_transfers", + "tableTo": "developers", + "columnsFrom": ["developer_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "developer_transfers_created_by_users_id_fk": { + "name": "developer_transfers_created_by_users_id_fk", + "tableFrom": "developer_transfers", + "tableTo": "users", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "developer_transfers_accepted_by_users_id_fk": { + "name": "developer_transfers_accepted_by_users_id_fk", + "tableFrom": "developer_transfers", + "tableTo": "users", + "columnsFrom": ["accepted_by"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "developers": { + "name": "developers", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "owner_user_id": { + "name": "owner_user_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "approved_at": { + "name": "approved_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'1970-01-01T00:00:00.000Z'" + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'1970-01-01T00:00:00.000Z'" + }, + "avatar_url": { + "name": "avatar_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "contact_email": { + "name": "contact_email", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "ownership_epoch": { + "name": "ownership_epoch", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "content_revision": { + "name": "content_revision", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "profile_generation": { + "name": "profile_generation", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "approved_revision": { + "name": "approved_revision", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "approved_by": { + "name": "approved_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "github_org_verified": { + "name": "github_org_verified", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "github_verification_note": { + "name": "github_verification_note", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "github_verified_at": { + "name": "github_verified_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "github_url_verified": { + "name": "github_url_verified", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "url_check_cooldown_until": { + "name": "url_check_cooldown_until", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "idx_developers_identifier_nocase": { + "name": "idx_developers_identifier_nocase", + "columns": ["lower(\"identifier\")"], + "isUnique": true + }, + "idx_developers_owner_unique": { + "name": "idx_developers_owner_unique", + "columns": ["owner_user_id"], + "isUnique": true + }, + "idx_developers_approved": { + "name": "idx_developers_approved", + "columns": ["approved_at"], + "isUnique": false + } + }, + "foreignKeys": { + "developers_owner_user_id_users_id_fk": { + "name": "developers_owner_user_id_users_id_fk", + "tableFrom": "developers", + "tableTo": "users", + "columnsFrom": ["owner_user_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": { + "developers_ownership_epoch_check": { + "name": "developers_ownership_epoch_check", + "value": "\"developers\".\"ownership_epoch\" >= 1" + }, + "developers_content_revision_check": { + "name": "developers_content_revision_check", + "value": "\"developers\".\"content_revision\" >= 1" + }, + "developers_github_org_verified_check": { + "name": "developers_github_org_verified_check", + "value": "\"developers\".\"github_org_verified\" IN (0, 1)" + }, + "developers_github_url_verified_check": { + "name": "developers_github_url_verified_check", + "value": "\"developers\".\"github_url_verified\" = 1" + } + } + }, + "extension_resource_usage": { + "name": "extension_resource_usage", + "columns": { + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "subject": { + "name": "subject", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes": { + "name": "bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "extensions": { + "name": "extensions", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "revisions": { + "name": "revisions", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + } + }, + "indexes": { + "idx_extension_resource_usage_subject": { + "name": "idx_extension_resource_usage_subject", + "columns": ["scope", "subject"], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "extension_revisions": { + "name": "extension_revisions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "extension_id": { + "name": "extension_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "developer_id": { + "name": "developer_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "submitted_by": { + "name": "submitted_by", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content_bytes": { + "name": "content_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "summary_name": { + "name": "summary_name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "summary_version": { + "name": "summary_version", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "summary_description": { + "name": "summary_description", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "content_readable": { + "name": "content_readable", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "content_hash": { + "name": "content_hash", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "compacted_at": { + "name": "compacted_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reviewer_id": { + "name": "reviewer_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "review_note": { + "name": "review_note", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + }, + "reviewed_at": { + "name": "reviewed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "ownership_epoch": { + "name": "ownership_epoch", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + } + }, + "indexes": { + "idx_extension_revisions_retention": { + "name": "idx_extension_revisions_retention", + "columns": ["compacted_at", "reviewed_at", "id", "status"], + "isUnique": false + }, + "idx_extension_revisions_submitted_by": { + "name": "idx_extension_revisions_submitted_by", + "columns": ["submitted_by"], + "isUnique": false + }, + "idx_extension_revisions_developer": { + "name": "idx_extension_revisions_developer", + "columns": ["developer_id"], + "isUnique": false + }, + "idx_extension_revisions_pending": { + "name": "idx_extension_revisions_pending", + "columns": ["extension_id"], + "isUnique": true, + "where": "\"extension_revisions\".\"status\" = 'pending'" + }, + "idx_extension_revisions_extension_page": { + "name": "idx_extension_revisions_extension_page", + "columns": ["extension_id", "\"created_at\" desc", "\"id\" desc"], + "isUnique": false + }, + "idx_extension_revisions_submitter_page": { + "name": "idx_extension_revisions_submitter_page", + "columns": ["submitted_by", "\"created_at\" desc", "\"id\" desc"], + "isUnique": false + }, + "idx_extension_revisions_queue_page": { + "name": "idx_extension_revisions_queue_page", + "columns": ["status", "created_at", "id"], + "isUnique": false + }, + "idx_extension_revisions_reviewed": { + "name": "idx_extension_revisions_reviewed", + "columns": ["extension_id", "reviewed_at"], + "isUnique": false, + "where": "\"extension_revisions\".\"status\" IN ('approved', 'rejected')" + }, + "idx_extension_revisions_submitter_pending": { + "name": "idx_extension_revisions_submitter_pending", + "columns": ["submitted_by"], + "isUnique": false, + "where": "\"extension_revisions\".\"status\" = 'pending'" + } + }, + "foreignKeys": { + "extension_revisions_extension_id_extensions_id_fk": { + "name": "extension_revisions_extension_id_extensions_id_fk", + "tableFrom": "extension_revisions", + "tableTo": "extensions", + "columnsFrom": ["extension_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "extension_revisions_submitted_by_users_id_fk": { + "name": "extension_revisions_submitted_by_users_id_fk", + "tableFrom": "extension_revisions", + "tableTo": "users", + "columnsFrom": ["submitted_by"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "extension_revisions_reviewer_id_users_id_fk": { + "name": "extension_revisions_reviewer_id_users_id_fk", + "tableFrom": "extension_revisions", + "tableTo": "users", + "columnsFrom": ["reviewer_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": { + "extension_revisions_status_check": { + "name": "extension_revisions_status_check", + "value": "\"extension_revisions\".\"status\" IN ('pending', 'approved', 'rejected')" + }, + "extension_revisions_ownership_epoch_check": { + "name": "extension_revisions_ownership_epoch_check", + "value": "\"extension_revisions\".\"ownership_epoch\" >= 1" + } + } + }, + "extension_write_events": { + "name": "extension_write_events", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "developer_id": { + "name": "developer_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "occurred_at": { + "name": "occurred_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_extension_write_events_account": { + "name": "idx_extension_write_events_account", + "columns": ["account_id", "occurred_at"], + "isUnique": false + }, + "idx_extension_write_events_developer": { + "name": "idx_extension_write_events_developer", + "columns": ["developer_id", "occurred_at"], + "isUnique": false + }, + "idx_extension_write_events_time": { + "name": "idx_extension_write_events_time", + "columns": ["occurred_at"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "extensions": { + "name": "extensions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "developer_id": { + "name": "developer_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "published_bytes": { + "name": "published_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "published_at": { + "name": "published_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "published_revision_id": { + "name": "published_revision_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "manifest": { + "name": "manifest", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "releases": { + "name": "releases", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "website": { + "name": "website", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "license": { + "name": "license", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "icon_url": { + "name": "icon_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "readme": { + "name": "readme", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "version": { + "name": "version", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "download_url": { + "name": "download_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + }, + "delisted_at": { + "name": "delisted_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "delist_reason": { + "name": "delist_reason", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "idx_extensions_published_revision": { + "name": "idx_extensions_published_revision", + "columns": ["published_revision_id"], + "isUnique": false + }, + "idx_extensions_identifier_nocase": { + "name": "idx_extensions_identifier_nocase", + "columns": ["lower(\"identifier\")"], + "isUnique": true + }, + "idx_extensions_developer_order": { + "name": "idx_extensions_developer_order", + "columns": ["developer_id", "lower(\"identifier\")", "id"], + "isUnique": false + }, + "idx_extensions_public_order": { + "name": "idx_extensions_public_order", + "columns": ["lower(\"identifier\")", "id"], + "isUnique": false, + "where": "\"extensions\".\"published_at\" IS NOT NULL AND \"extensions\".\"delisted_at\" IS NULL" + }, + "idx_extensions_type_public_order": { + "name": "idx_extensions_type_public_order", + "columns": ["type", "lower(\"identifier\")", "id"], + "isUnique": false, + "where": "\"extensions\".\"published_at\" IS NOT NULL AND \"extensions\".\"delisted_at\" IS NULL" + } + }, + "foreignKeys": { + "extensions_developer_id_developers_id_fk": { + "name": "extensions_developer_id_developers_id_fk", + "tableFrom": "extensions", + "tableTo": "developers", + "columnsFrom": ["developer_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": { + "extensions_published_content_check": { + "name": "extensions_published_content_check", + "value": "\"extensions\".\"published_at\" IS NULL OR (\n \"extensions\".\"type\" IS NOT NULL AND \"extensions\".\"name\" IS NOT NULL AND\n \"extensions\".\"description\" IS NOT NULL AND \"extensions\".\"releases\" IS NOT NULL AND\n \"extensions\".\"website\" IS NOT NULL AND \"extensions\".\"license\" IS NOT NULL AND\n \"extensions\".\"readme\" IS NOT NULL AND \"extensions\".\"source\" IS NOT NULL AND\n \"extensions\".\"version\" IS NOT NULL AND \"extensions\".\"download_url\" IS NOT NULL\n )" + } + } + }, + "m2m_publisher_clients": { + "name": "m2m_publisher_clients", + "columns": { + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "developer_id": { + "name": "developer_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "can_publish": { + "name": "can_publish", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "ownership_epoch": { + "name": "ownership_epoch", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + }, + "revoked_at": { + "name": "revoked_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "idx_m2m_publisher_clients_developer": { + "name": "idx_m2m_publisher_clients_developer", + "columns": ["developer_id"], + "isUnique": false + } + }, + "foreignKeys": { + "m2m_publisher_clients_developer_id_developers_id_fk": { + "name": "m2m_publisher_clients_developer_id_developers_id_fk", + "tableFrom": "m2m_publisher_clients", + "tableTo": "developers", + "columnsFrom": ["developer_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": { + "m2m_publisher_clients_can_publish_check": { + "name": "m2m_publisher_clients_can_publish_check", + "value": "\"m2m_publisher_clients\".\"can_publish\" IN (0, 1)" + }, + "m2m_publisher_clients_ownership_epoch_check": { + "name": "m2m_publisher_clients_ownership_epoch_check", + "value": "\"m2m_publisher_clients\".\"ownership_epoch\" >= 1" + } + } + }, + "package_artifacts": { + "name": "package_artifacts", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "manifest": { + "name": "manifest", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "manifest_hash": { + "name": "manifest_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sha256": { + "name": "sha256", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "object_id": { + "name": "object_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "download_url": { + "name": "download_url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "published_at": { + "name": "published_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + } + }, + "indexes": { + "idx_package_artifact_manifest": { + "name": "idx_package_artifact_manifest", + "columns": ["source_id", "manifest_hash"], + "isUnique": true + }, + "idx_package_artifacts_object": { + "name": "idx_package_artifacts_object", + "columns": ["object_id"], + "isUnique": true + } + }, + "foreignKeys": { + "package_artifacts_source_id_package_sources_id_fk": { + "name": "package_artifacts_source_id_package_sources_id_fk", + "tableFrom": "package_artifacts", + "tableTo": "package_sources", + "columnsFrom": ["source_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "package_artifacts_object_id_package_objects_id_fk": { + "name": "package_artifacts_object_id_package_objects_id_fk", + "tableFrom": "package_artifacts", + "tableTo": "package_objects", + "columnsFrom": ["object_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "package_icons": { + "name": "package_icons", + "columns": { + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "avif_object_id": { + "name": "avif_object_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "avif_sha256": { + "name": "avif_sha256", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "png_object_id": { + "name": "png_object_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "png_sha256": { + "name": "png_sha256", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "svg_object_id": { + "name": "svg_object_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "svg_sha256": { + "name": "svg_sha256", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "package_icons_source_id_package_sources_id_fk": { + "name": "package_icons_source_id_package_sources_id_fk", + "tableFrom": "package_icons", + "tableTo": "package_sources", + "columnsFrom": ["source_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "package_icons_avif_object_id_package_objects_id_fk": { + "name": "package_icons_avif_object_id_package_objects_id_fk", + "tableFrom": "package_icons", + "tableTo": "package_objects", + "columnsFrom": ["avif_object_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "package_icons_png_object_id_package_objects_id_fk": { + "name": "package_icons_png_object_id_package_objects_id_fk", + "tableFrom": "package_icons", + "tableTo": "package_objects", + "columnsFrom": ["png_object_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "package_icons_svg_object_id_package_objects_id_fk": { + "name": "package_icons_svg_object_id_package_objects_id_fk", + "tableFrom": "package_icons", + "tableTo": "package_objects", + "columnsFrom": ["svg_object_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "package_jobs": { + "name": "package_jobs", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "fingerprint": { + "name": "fingerprint", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "owner_user_id": { + "name": "owner_user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "owner": { + "name": "owner", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "manifest": { + "name": "manifest", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "upload_object_id": { + "name": "upload_object_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "upload_sha256": { + "name": "upload_sha256", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'queued'" + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "lease_id": { + "name": "lease_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "lease_until": { + "name": "lease_until", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "artifact_id": { + "name": "artifact_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error_code": { + "name": "error_code", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error_status": { + "name": "error_status", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "enqueued_at": { + "name": "enqueued_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + } + }, + "indexes": { + "idx_package_jobs_fingerprint": { + "name": "idx_package_jobs_fingerprint", + "columns": ["fingerprint"], + "isUnique": true, + "where": "\"package_jobs\".\"status\" != 'failed'" + }, + "idx_package_jobs_owner": { + "name": "idx_package_jobs_owner", + "columns": ["owner_user_id"], + "isUnique": false + }, + "idx_package_jobs_upload_object": { + "name": "idx_package_jobs_upload_object", + "columns": ["upload_object_id"], + "isUnique": false + }, + "idx_package_jobs_dispatch": { + "name": "idx_package_jobs_dispatch", + "columns": ["status", "enqueued_at"], + "isUnique": false + } + }, + "foreignKeys": { + "package_jobs_upload_object_id_package_objects_id_fk": { + "name": "package_jobs_upload_object_id_package_objects_id_fk", + "tableFrom": "package_jobs", + "tableTo": "package_objects", + "columnsFrom": ["upload_object_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "package_jobs_artifact_id_package_artifacts_id_fk": { + "name": "package_jobs_artifact_id_package_artifacts_id_fk", + "tableFrom": "package_jobs", + "tableTo": "package_artifacts", + "columnsFrom": ["artifact_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": { + "package_jobs_status_check": { + "name": "package_jobs_status_check", + "value": "\"package_jobs\".\"status\" IN ('queued','processing','ready','failed')" + } + } + }, + "package_objects": { + "name": "package_objects", + "columns": { + "deleting": { + "name": "deleting", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes": { + "name": "bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + } + }, + "indexes": { + "idx_package_objects_creator": { + "name": "idx_package_objects_creator", + "columns": ["created_by"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "package_sources": { + "name": "package_sources", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "profile_generation": { + "name": "profile_generation", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "extension_identifier": { + "name": "extension_identifier", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "version": { + "name": "version", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sha256": { + "name": "sha256", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "object_id": { + "name": "object_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + } + }, + "indexes": { + "idx_package_source_version": { + "name": "idx_package_source_version", + "columns": ["profile_generation", "extension_identifier", "version"], + "isUnique": true + }, + "idx_package_sources_extension": { + "name": "idx_package_sources_extension", + "columns": ["extension_identifier"], + "isUnique": false + }, + "idx_package_sources_object": { + "name": "idx_package_sources_object", + "columns": ["object_id"], + "isUnique": true + } + }, + "foreignKeys": { + "package_sources_object_id_package_objects_id_fk": { + "name": "package_sources_object_id_package_objects_id_fk", + "tableFrom": "package_sources", + "tableTo": "package_objects", + "columnsFrom": ["object_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "users": { + "name": "users", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "email_verified": { + "name": "email_verified", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "picture": { + "name": "picture", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "is_moderator": { + "name": "is_moderator", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "github_login": { + "name": "github_login", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "github_orgs": { + "name": "github_orgs", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "github_orgs_expires_at": { + "name": "github_orgs_expires_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": { + "\"package_sources\".\"extension_id\"": "\"package_sources\".\"extension_identifier\"" + } + }, + "internal": { + "indexes": { + "idx_developers_identifier_nocase": { + "columns": { + "lower(\"identifier\")": { + "isExpression": true + } + } + }, + "idx_extension_revisions_extension_page": { + "columns": { + "\"created_at\" desc": { + "isExpression": true + }, + "\"id\" desc": { + "isExpression": true + } + } + }, + "idx_extension_revisions_submitter_page": { + "columns": { + "\"created_at\" desc": { + "isExpression": true + }, + "\"id\" desc": { + "isExpression": true + } + } + }, + "idx_extensions_identifier_nocase": { + "columns": { + "lower(\"identifier\")": { + "isExpression": true + } + } + }, + "idx_extensions_developer_order": { + "columns": { + "lower(\"identifier\")": { + "isExpression": true + } + } + }, + "idx_extensions_public_order": { + "columns": { + "lower(\"identifier\")": { + "isExpression": true + } + } + }, + "idx_extensions_type_public_order": { + "columns": { + "lower(\"identifier\")": { + "isExpression": true + } + } + } + } + } +} diff --git a/src/services/extensions/v2/db/migrations/meta/_journal.json b/src/services/extensions/v2/db/migrations/meta/_journal.json index a3a4fe24..e00111b6 100644 --- a/src/services/extensions/v2/db/migrations/meta/_journal.json +++ b/src/services/extensions/v2/db/migrations/meta/_journal.json @@ -127,6 +127,13 @@ "when": 1793931547450, "tag": "0031_package_jobs", "breakpoints": true + }, + { + "idx": 32, + "version": "6", + "when": 1794068614832, + "tag": "0032_directory_identity", + "breakpoints": true } ] } diff --git a/src/services/extensions/v2/db/packages.ts b/src/services/extensions/v2/db/packages.ts index 1c333a19..413a85ce 100644 --- a/src/services/extensions/v2/db/packages.ts +++ b/src/services/extensions/v2/db/packages.ts @@ -39,7 +39,7 @@ export interface ArtifactRow { download_url: string; published_at: string | null; created_at: string; - extension_id: string; + extension_identifier: string; profile_generation: string; version: string; icon_source_id: string | null; @@ -51,7 +51,7 @@ export interface ArtifactRow { svg_sha256: string | null; } -export const ARTIFACT_SELECT = `SELECT a.*, s.extension_id, s.profile_generation, s.version, i.source_id AS icon_source_id, i.avif_object_id, i.avif_sha256, i.png_object_id, i.png_sha256, i.svg_object_id, i.svg_sha256 +export const ARTIFACT_SELECT = `SELECT a.*, s.extension_identifier, s.profile_generation, s.version, i.source_id AS icon_source_id, i.avif_object_id, i.avif_sha256, i.png_object_id, i.png_sha256, i.svg_object_id, i.svg_sha256 FROM package_artifacts a JOIN package_sources s ON s.id=a.source_id LEFT JOIN package_icons i ON i.source_id=s.id`; // All submission, publisher, correction and approval writers use this check. @@ -66,7 +66,7 @@ export async function validatePackageContent( if (!developerId) { const ref = await db .prepare( - `SELECT d.id FROM extensions e JOIN developers d ON d.id=e.developer_id WHERE lower(e.id)=lower(?) AND (? IS NULL OR d.owner_user_id=?)` + `SELECT d.id FROM extensions e JOIN developers d ON d.id=e.developer_id WHERE lower(e.identifier)=lower(?) AND (? IS NULL OR d.owner_user_id=?)` ) .bind(id, callerId ?? null, callerId ?? null) .first<{ id: string }>(); @@ -75,7 +75,7 @@ export async function validatePackageContent( } const published = await db .prepare( - `SELECT DISTINCT s.version FROM package_artifacts a JOIN package_sources s ON s.id=a.source_id WHERE s.extension_id=? AND a.published_at IS NOT NULL` + `SELECT DISTINCT s.version FROM package_artifacts a JOIN package_sources s ON s.id=a.source_id WHERE s.extension_identifier=? AND a.published_at IS NOT NULL` ) .bind(id) .all<{ version: string }>(); @@ -122,7 +122,7 @@ export async function validatePackageContent( ]; const rows = await db .prepare( - `${ARTIFACT_SELECT} JOIN developers d ON d.profile_generation=s.profile_generation WHERE a.id IN (SELECT value FROM json_each(?)) AND s.extension_id=? AND d.id=?` + `${ARTIFACT_SELECT} JOIN developers d ON d.profile_generation=s.profile_generation WHERE a.id IN (SELECT value FROM json_each(?)) AND s.extension_identifier=? AND d.id=?` ) .bind(JSON.stringify(artifactIds), id, developerId) .all(); diff --git a/src/services/extensions/v2/db/revisions.ts b/src/services/extensions/v2/db/revisions.ts index ba1983c2..bce5489f 100644 --- a/src/services/extensions/v2/db/revisions.ts +++ b/src/services/extensions/v2/db/revisions.ts @@ -53,6 +53,7 @@ const revisionCursor = createCursorCodec( interface RevisionRow { id: string; extensionId: string; + extensionIdentifier: string; developerId: string; submittedBy: string; status: string; @@ -86,6 +87,7 @@ function parseRevisionRow(row: RevisionRow): StoredRevision { const REVISION_COLUMNS = { id: extensionRevisions.id, extensionId: extensionRevisions.extensionId, + extensionIdentifier: sql`(SELECT identifier FROM extensions WHERE id=${extensionRevisions.extensionId})`, developerId: extensionRevisions.developerId, submittedBy: extensionRevisions.submittedBy, status: extensionRevisions.status, @@ -114,6 +116,7 @@ function parseSummaryRow(row: SummaryRow): ExtensionRevisionSummary { return { id: row.id, extension_id: row.extensionId, + extension_identifier: row.extensionIdentifier, developer_id: row.developerId, submitted_by: row.submittedBy, status: row.status as RevisionStatus, @@ -164,7 +167,7 @@ export class ExtensionRevisionsDatabase { SELECT ${id}, e.id, d.id, ${input.callerId}, 'pending', ${JSON.stringify(input.content)}, d.ownership_epoch FROM ${extensions} e JOIN ${developers} d ON d.id = e.developer_id - WHERE LOWER(e.id) = LOWER(${input.extensionId}) + WHERE LOWER(e.identifier) = LOWER(${input.extensionId}) AND d.owner_user_id = ${input.callerId} AND EXISTS ( SELECT 1 FROM ${users} u @@ -199,7 +202,7 @@ export class ExtensionRevisionsDatabase { revisionId: string; callerId: string; content: ExtensionContent; - }): Promise> { + }): Promise> { const id = crypto.randomUUID(); try { const packageError = await validatePackageContent( @@ -210,7 +213,7 @@ export class ExtensionRevisionsDatabase { input.callerId ); if (packageError) return { data: null, error: packageError }; - const results = await this.db.$client.batch<{ extensionId: string }>([ + const results = await this.db.$client.batch<{ identifier: string }>([ this.db.$client .prepare( ` @@ -219,7 +222,7 @@ export class ExtensionRevisionsDatabase { AND EXISTS ( SELECT 1 FROM extensions e JOIN developers d ON d.id = e.developer_id JOIN users u ON u.id = d.owner_user_id - WHERE LOWER(e.id) = LOWER(?) AND e.id = extension_revisions.extension_id + WHERE LOWER(e.identifier) = LOWER(?) AND e.id = extension_revisions.extension_id AND d.id = extension_revisions.developer_id AND d.ownership_epoch = extension_revisions.ownership_epoch AND d.owner_user_id = ? AND u.deleted_at IS NULL @@ -244,8 +247,8 @@ export class ExtensionRevisionsDatabase { (id, extension_id, developer_id, submitted_by, status, content, ownership_epoch) SELECT ?, e.id, d.id, ?, 'pending', ?, d.ownership_epoch FROM extensions e JOIN developers d ON d.id = e.developer_id - WHERE LOWER(e.id) = LOWER(?) AND changes() = 1 - RETURNING extension_id AS extensionId + WHERE LOWER(e.identifier) = LOWER(?) AND changes() = 1 + RETURNING (SELECT identifier FROM extensions WHERE id=extension_id) AS identifier ` ) .bind( @@ -258,7 +261,10 @@ export class ExtensionRevisionsDatabase { const [replacement] = results[1].results; if (replacement) return { - data: { id, extensionId: replacement.extensionId }, + data: { + id, + identifier: replacement.identifier + }, error: null }; const error = await this.proposeBlockedError(input); @@ -291,7 +297,9 @@ export class ExtensionRevisionsDatabase { .select({ ownerUserId: developers.ownerUserId }) .from(extensions) .innerJoin(developers, eq(extensions.developerId, developers.id)) - .where(sql`LOWER(${extensions.id}) = LOWER(${input.extensionId})`); + .where( + sql`LOWER(${extensions.identifier}) = LOWER(${input.extensionId})` + ); if (!existing) { return { @@ -311,7 +319,7 @@ export class ExtensionRevisionsDatabase { .from(extensionRevisions) .where( and( - sql`LOWER(${extensionRevisions.extensionId}) = LOWER(${input.extensionId})`, + sql`${extensionRevisions.extensionId} = (SELECT id FROM extensions WHERE lower(identifier)=lower(${input.extensionId}))`, eq(extensionRevisions.status, "pending") ) ); @@ -357,7 +365,7 @@ export class ExtensionRevisionsDatabase { SELECT ${id}, e.id, d.id, ${input.submittedBy}, 'pending', ${JSON.stringify(input.content)}, d.ownership_epoch FROM ${extensions} e JOIN ${developers} d ON d.id = e.developer_id - WHERE LOWER(e.id) = LOWER(${input.extensionId}) + WHERE LOWER(e.identifier) = LOWER(${input.extensionId}) AND d.id = ${input.developerId} AND EXISTS ( SELECT 1 FROM ${users} u @@ -452,7 +460,9 @@ export class ExtensionRevisionsDatabase { const [existing] = await this.db .select({ one: sql`1` }) .from(extensions) - .where(sql`LOWER(${extensions.id}) = LOWER(${input.extensionId})`); + .where( + sql`LOWER(${extensions.identifier}) = LOWER(${input.extensionId})` + ); if (!existing) { return { @@ -466,7 +476,7 @@ export class ExtensionRevisionsDatabase { .from(extensionRevisions) .where( and( - sql`LOWER(${extensionRevisions.extensionId}) = LOWER(${input.extensionId})`, + sql`${extensionRevisions.extensionId} = (SELECT id FROM extensions WHERE lower(identifier)=lower(${input.extensionId}))`, eq(extensionRevisions.status, "pending") ) ); @@ -516,7 +526,7 @@ export class ExtensionRevisionsDatabase { SELECT 1 FROM users u WHERE u.id=${readerId} AND u.deleted_at IS NULL AND (u.is_moderator=1 OR (${extensionId ?? null} IS NOT NULL AND EXISTS ( SELECT 1 FROM extensions e JOIN developers d ON d.id=e.developer_id - WHERE e.id=${extensionId ?? null} AND d.owner_user_id=${readerId} + WHERE lower(e.identifier)=lower(${extensionId ?? null}) AND d.owner_user_id=${readerId} ))) )`); if (decoded) { @@ -584,7 +594,9 @@ export class ExtensionRevisionsDatabase { : "asc"; const conditions: SQL[] = []; if (filters.extensionId) { - conditions.push(eq(extensionRevisions.extensionId, filters.extensionId)); + conditions.push( + sql`${extensionRevisions.extensionId} = (SELECT id FROM extensions WHERE lower(identifier)=lower(${filters.extensionId}))` + ); } if (filters.status) { conditions.push(eq(extensionRevisions.status, filters.status)); @@ -650,7 +662,7 @@ export class ExtensionRevisionsDatabase { .where( and( eq(extensionRevisions.id, id), - sql`LOWER(${extensionRevisions.extensionId}) = LOWER(${extensionId})`, + sql`${extensionRevisions.extensionId} = (SELECT id FROM extensions WHERE lower(identifier)=lower(${extensionId}))`, readerId ? sql`EXISTS ( SELECT 1 FROM users u WHERE u.id=${readerId} AND u.deleted_at IS NULL @@ -729,7 +741,7 @@ export class ExtensionRevisionsDatabase { .where( and( eq(extensionRevisions.id, id), - sql`LOWER(${extensionRevisions.extensionId}) = LOWER(${extensionId})`, + sql`${extensionRevisions.extensionId} = (SELECT id FROM extensions WHERE lower(identifier)=lower(${extensionId}))`, eq(extensionRevisions.status, "pending"), sql`EXISTS ( SELECT 1 FROM ${users} @@ -808,7 +820,7 @@ export class ExtensionRevisionsDatabase { try { const packageError = await validatePackageContent( this.db.$client, - revision.extension_id, + revision.extension_identifier, revision.developer_id, parsed.data ); diff --git a/src/services/extensions/v2/db/schema.ts b/src/services/extensions/v2/db/schema.ts index b260b94b..53630359 100644 --- a/src/services/extensions/v2/db/schema.ts +++ b/src/services/extensions/v2/db/schema.ts @@ -42,6 +42,7 @@ export const extensions = sqliteTable( "extensions", { id: text("id").primaryKey(), + identifier: text("identifier").notNull(), developerId: text("developer_id") .notNull() .references(() => developers.id), @@ -86,11 +87,10 @@ export const extensions = sqliteTable( }, (table) => [ index("idx_extensions_published_revision").on(table.publishedRevisionId), - // Case-insensitive id uniqueness. The id is a lowercase slug by schema, - // but adopted rows predate that, and this is what stops two developers - // racing for ids that differ only in case — the job migration 0011's - // extension_submissions.target_key index used to do from the other side. - uniqueIndex("idx_extensions_id_nocase").on(sql`lower(${table.id})`), + // Public identifiers are globally unique, independently of UUID identity. + uniqueIndex("idx_extensions_identifier_nocase").on( + sql`lower(${table.identifier})` + ), // Not partial, unlike the two below: this one serves both the public // developer_id filter and GET /extensions/mine, which pages every owned // extension and so cannot filter on published_at. A partial index would @@ -99,19 +99,19 @@ export const extensions = sqliteTable( // same index and checks published_at per row. index("idx_extensions_developer_order").on( table.developerId, - sql`lower(${table.id})`, + sql`lower(${table.identifier})`, table.id ), // These two are partial: every read that uses them filters on // published_at IS NOT NULL AND delisted_at IS NULL, so an unpublished or // delisted row would only bloat the index the extension directory scans. index("idx_extensions_public_order") - .on(sql`lower(${table.id})`, table.id) + .on(sql`lower(${table.identifier})`, table.id) .where( sql`${table.publishedAt} IS NOT NULL AND ${table.delistedAt} IS NULL` ), index("idx_extensions_type_public_order") - .on(table.type, sql`lower(${table.id})`, table.id) + .on(table.type, sql`lower(${table.identifier})`, table.id) .where( sql`${table.publishedAt} IS NOT NULL AND ${table.delistedAt} IS NULL` ), @@ -144,21 +144,13 @@ export const developers = sqliteTable( "developers", { id: text("id").primaryKey(), + identifier: text("identifier").notNull(), type: text("type").notNull(), name: text("name").notNull(), url: text("url"), ownerUserId: text("owner_user_id").references(() => users.id), approvedAt: text("approved_at"), - // Placeholder default from migration 0002 (SQLite rejects non-constant - // ALTER TABLE ADD COLUMN defaults). Every write sets this explicitly (see - // db/developer-profiles.ts) - the literal default is never actually read, - // but it is part of the real column definition, so it is kept here for - // baseline-diff fidelity against the existing database. - // - // Replacing it needs a table rebuild, which this table cannot have: three - // tables reference it and a parent cannot be dropped with foreign keys - // enforced (see migration 0021's header). Rebuilding all three children - // too is a lot of risk for a default nothing reads. + // Kept in sync with the original constant database default. Writers set both timestamps. createdAt: text("created_at").notNull().default("1970-01-01T00:00:00.000Z"), updatedAt: text("updated_at").notNull().default("1970-01-01T00:00:00.000Z"), avatarUrl: text("avatar_url"), @@ -189,6 +181,9 @@ export const developers = sqliteTable( urlCheckCooldownUntil: text("url_check_cooldown_until") }, (table) => [ + uniqueIndex("idx_developers_identifier_nocase").on( + sql`lower(${table.identifier})` + ), uniqueIndex("idx_developers_owner_unique").on(table.ownerUserId), index("idx_developers_approved").on(table.approvedAt), check( @@ -535,7 +530,7 @@ export const packageSources = sqliteTable( { id: text("id").primaryKey(), profileGeneration: text("profile_generation").notNull(), - extensionId: text("extension_id").notNull(), + extensionIdentifier: text("extension_identifier").notNull(), version: text("version").notNull(), sha256: text("sha256").notNull(), objectId: text("object_id") @@ -548,10 +543,10 @@ export const packageSources = sqliteTable( (table) => [ uniqueIndex("idx_package_source_version").on( table.profileGeneration, - table.extensionId, + table.extensionIdentifier, table.version ), - index("idx_package_sources_extension").on(table.extensionId), + index("idx_package_sources_extension").on(table.extensionIdentifier), uniqueIndex("idx_package_sources_object").on(table.objectId) ] ); diff --git a/src/services/extensions/v2/email/notify.ts b/src/services/extensions/v2/email/notify.ts index c67cd5e1..94126cac 100644 --- a/src/services/extensions/v2/email/notify.ts +++ b/src/services/extensions/v2/email/notify.ts @@ -62,6 +62,7 @@ export async function sendModerationNotification( return false; } developerName = developerEmail?.developerName; + developerId = developerEmail?.developerIdentifier; } else if (input.extensionId) { const recipient = await resolveExtensionEmail(db, input.extensionId); if (!recipient) { @@ -73,7 +74,7 @@ export async function sendModerationNotification( } to = recipient.to; extensionName = recipient.extensionName; - developerId = recipient.developerId; + developerId = recipient.developerIdentifier; developerName = recipient.developerName; } else if (input.developerId) { const recipient = await resolveDeveloperProfileEmail( @@ -89,6 +90,7 @@ export async function sendModerationNotification( } to = recipient.to; developerName = recipient.developerName; + developerId = recipient.developerIdentifier; } else { logError("email", "Notification missing extension and developer", { kind: input.kind diff --git a/src/services/extensions/v2/email/recipients.ts b/src/services/extensions/v2/email/recipients.ts index c92d3243..d50f9d8e 100644 --- a/src/services/extensions/v2/email/recipients.ts +++ b/src/services/extensions/v2/email/recipients.ts @@ -32,14 +32,14 @@ export async function resolveExtensionEmail( ): Promise<{ to: string; extensionName?: string; - developerId: string; + developerIdentifier: string; developerName?: string; } | null> { try { const [row] = await db .select({ extensionName: extensions.name, - developerId: extensions.developerId, + developerIdentifier: developers.identifier, developerName: developers.name, contactEmail: developers.contactEmail, accountEmail: users.email @@ -50,7 +50,7 @@ export async function resolveExtensionEmail( // Moderation routes pass the raw URL param, and ids are matched // case-insensitively everywhere else (LOWER(id) = LOWER(id)) - mixed // case is a supported input, not a miss. - .where(sql`LOWER(${extensions.id}) = LOWER(${extensionId})`); + .where(sql`LOWER(${extensions.identifier}) = LOWER(${extensionId})`); if (!row) return null; @@ -60,7 +60,7 @@ export async function resolveExtensionEmail( return { to, extensionName: row.extensionName ?? undefined, - developerId: row.developerId, + developerIdentifier: row.developerIdentifier, developerName: row.developerName }; } catch { @@ -71,10 +71,15 @@ export async function resolveExtensionEmail( export async function resolveDeveloperProfileEmail( db: ExtensionsDb, developerId: string -): Promise<{ to: string; developerName?: string } | null> { +): Promise<{ + to: string; + developerIdentifier: string; + developerName?: string; +} | null> { try { const [row] = await db .select({ + developerIdentifier: developers.identifier, developerName: developers.name, contactEmail: developers.contactEmail, accountEmail: users.email @@ -87,7 +92,11 @@ export async function resolveDeveloperProfileEmail( const to = pickEmail(row.contactEmail, row.accountEmail); if (!to) return null; - return { to, developerName: row.developerName }; + return { + to, + developerIdentifier: row.developerIdentifier, + developerName: row.developerName + }; } catch { return null; } diff --git a/src/services/extensions/v2/packages/build.ts b/src/services/extensions/v2/packages/build.ts index 7dbdf997..89190457 100644 --- a/src/services/extensions/v2/packages/build.ts +++ b/src/services/extensions/v2/packages/build.ts @@ -26,7 +26,7 @@ export const MAX_PACKAGE_STORAGE_BYTES = 512 * 1024 * 1024; export const MAX_PACKAGE_OBJECTS = 256; function ownerPredicate() { - return `EXISTS (SELECT 1 FROM developers d JOIN users u ON u.id=d.owner_user_id WHERE d.id=? AND d.profile_generation=? AND d.ownership_epoch=? AND d.owner_user_id=? AND u.deleted_at IS NULL AND NOT EXISTS (SELECT 1 FROM extensions e WHERE lower(e.id)=lower(?) AND e.developer_id!=d.id) AND (? IS NULL OR EXISTS (SELECT 1 FROM m2m_publisher_clients p WHERE p.client_id=? AND p.developer_id=d.id AND p.ownership_epoch=d.ownership_epoch AND p.revoked_at IS NULL)))`; + return `EXISTS (SELECT 1 FROM developers d JOIN users u ON u.id=d.owner_user_id WHERE d.id=? AND d.profile_generation=? AND d.ownership_epoch=? AND d.owner_user_id=? AND u.deleted_at IS NULL AND NOT EXISTS (SELECT 1 FROM extensions e WHERE lower(e.identifier)=lower(?) AND e.developer_id!=d.id) AND (? IS NULL OR EXISTS (SELECT 1 FROM m2m_publisher_clients p WHERE p.client_id=? AND p.developer_id=d.id AND p.ownership_epoch=d.ownership_epoch AND p.revoked_at IS NULL)))`; } function ownerParams(owner: PackageOwner, id: string) { return [ @@ -48,7 +48,7 @@ export async function packageOwner( ) { const owner = await db .prepare( - `SELECT d.id,d.profile_generation,d.ownership_epoch,d.owner_user_id FROM developers d JOIN users u ON u.id=d.owner_user_id WHERE (? IS NULL AND d.owner_user_id=? OR d.id=? AND d.ownership_epoch=? AND EXISTS (SELECT 1 FROM m2m_publisher_clients p WHERE p.client_id=? AND p.developer_id=d.id AND p.ownership_epoch=d.ownership_epoch AND p.revoked_at IS NULL)) AND u.deleted_at IS NULL AND NOT EXISTS (SELECT 1 FROM extensions e WHERE lower(e.id)=lower(?) AND e.developer_id!=d.id)` + `SELECT d.id,d.profile_generation,d.ownership_epoch,d.owner_user_id FROM developers d JOIN users u ON u.id=d.owner_user_id WHERE (? IS NULL AND d.owner_user_id=? OR d.id=? AND d.ownership_epoch=? AND EXISTS (SELECT 1 FROM m2m_publisher_clients p WHERE p.client_id=? AND p.developer_id=d.id AND p.ownership_epoch=d.ownership_epoch AND p.revoked_at IS NULL)) AND u.deleted_at IS NULL AND NOT EXISTS (SELECT 1 FROM extensions e WHERE lower(e.identifier)=lower(?) AND e.developer_id!=d.id)` ) .bind( publisher?.clientId ?? null, @@ -155,7 +155,7 @@ export async function buildStoredPackage(input: { const { db, store, owner, manifest } = input; let source = await db .prepare( - "SELECT id,sha256,object_id FROM package_sources WHERE profile_generation=? AND extension_id=? AND version=?" + "SELECT id,sha256,object_id FROM package_sources WHERE profile_generation=? AND extension_identifier=? AND version=?" ) .bind(owner.profile_generation, manifest.id, manifest.version) .first(); @@ -234,7 +234,7 @@ export async function buildStoredPackage(input: { const sourceId = crypto.randomUUID(); await db .prepare( - `INSERT INTO package_sources (id,profile_generation,extension_id,version,sha256,object_id) + `INSERT INTO package_sources (id,profile_generation,extension_identifier,version,sha256,object_id) SELECT ?,?,?,?,?,? WHERE ${ownerPredicate()} AND EXISTS (SELECT 1 FROM package_objects WHERE id=? AND deleting=0) ON CONFLICT DO NOTHING` ) @@ -251,7 +251,7 @@ export async function buildStoredPackage(input: { .run(); source = await db .prepare( - "SELECT id,sha256,object_id FROM package_sources WHERE profile_generation=? AND extension_id=? AND version=?" + "SELECT id,sha256,object_id FROM package_sources WHERE profile_generation=? AND extension_identifier=? AND version=?" ) .bind(owner.profile_generation, manifest.id, manifest.version) .first(); diff --git a/src/services/extensions/v2/packages/public-assets.ts b/src/services/extensions/v2/packages/public-assets.ts index 40d609cb..9d315840 100644 --- a/src/services/extensions/v2/packages/public-assets.ts +++ b/src/services/extensions/v2/packages/public-assets.ts @@ -1,7 +1,6 @@ import { logError } from "../../../../lib/logger"; import { revalidateExtensions } from "../revalidate"; import type { ExtensionContent } from "../schemas/extensions"; -import { sha256 } from "../package-builder"; import { AwsClient } from "aws4fetch"; import type { Context } from "hono"; import { getPlatform } from "../../../../lib/middleware"; @@ -31,17 +30,17 @@ export async function publishExtensionIcon( return value; }; const row = await c.env.DB_EXTENSIONS.prepare( - "SELECT releases,published_revision_id FROM extensions WHERE id=? AND published_at IS NOT NULL AND delisted_at IS NULL" + "SELECT id,releases,published_revision_id FROM extensions WHERE lower(identifier)=? AND published_at IS NOT NULL AND delisted_at IS NULL" ) .bind(extensionId) - .first<{ releases: string; published_revision_id: string }>(); + .first<{ id: string; releases: string; published_revision_id: string }>(); if (!row) return; const latest = latestRelease( JSON.parse(row.releases) as ExtensionContent["releases"] ); if (!latest?.artifact_id) return; const artifact = await c.env.DB_EXTENSIONS.prepare( - `${ARTIFACT_SELECT} WHERE a.id=? AND a.published_at IS NOT NULL AND s.extension_id=?` + `${ARTIFACT_SELECT} WHERE a.id=? AND a.published_at IS NOT NULL AND s.extension_identifier=?` ) .bind(latest.artifact_id, extensionId) .first(); @@ -79,9 +78,6 @@ export async function publishExtensionIcon( "Invalid asset bucket name", 503 ); - const prefix = platform.getEnv("EXTENSION_ASSETS_S3_PREFIX") ?? "local"; - if (!/^[a-zA-Z0-9_-]+$/.test(prefix)) - throw new PackageError("ASSETS_UNAVAILABLE", "Invalid asset prefix", 503); const aws = new AwsClient({ accessKeyId: get("S3_ACCESS_KEY_ID"), secretAccessKey: get("S3_SECRET_ACCESS_KEY"), @@ -89,15 +85,8 @@ export async function publishExtensionIcon( service: "s3", retries: 0 }); - const identity = - artifact.svg_sha256 ?? - (await sha256( - new Uint8Array( - new TextEncoder().encode( - `${artifact.avif_sha256}:${artifact.png_sha256}` - ) - ) - )); + // The icon shares its immutable source UUID across formats and manifest revisions. + const assetId = artifact.icon_source_id; let iconUrl: string | undefined; for (const format of artifact.svg_object_id ? (["svg"] as const) @@ -110,7 +99,7 @@ export async function publishExtensionIcon( "Prepared asset is unavailable", 503 ); - const objectKey = `${prefix}/icons/${identity}.${format}`; + const objectKey = `extensions/${row.id}/icons/${assetId}.${format}`; const url = new URL(`${publicBase.href.replace(/\/$/, "")}/${objectKey}`) .href; const contentType = format === "svg" ? "image/svg+xml" : `image/${format}`; @@ -176,7 +165,7 @@ export async function publishExtensionIcon( } if (iconUrl) { const updated = await c.env.DB_EXTENSIONS.prepare( - "UPDATE extensions SET icon_url=? WHERE id=? AND published_revision_id=? AND delisted_at IS NULL AND icon_url IS NOT ?" + "UPDATE extensions SET icon_url=? WHERE lower(identifier)=? AND published_revision_id=? AND delisted_at IS NULL AND icon_url IS NOT ?" ) .bind(iconUrl, extensionId, row.published_revision_id, iconUrl) .run(); diff --git a/src/services/extensions/v2/routes/badges.ts b/src/services/extensions/v2/routes/badges.ts index 773af42d..b6a6b828 100644 --- a/src/services/extensions/v2/routes/badges.ts +++ b/src/services/extensions/v2/routes/badges.ts @@ -34,7 +34,7 @@ export function registerBadgeRoutes(app: ExtensionsV2App): void { const db = new ExtensionBadgesDatabase( getExtensionsDb(c.env.DB_EXTENSIONS) ); - const { data, error } = await db.getById(id); + const { data, error } = await db.getByIdentifier(id); if (error || !data) return c.json( errorBody(error, "Unable to load badge"), diff --git a/src/services/extensions/v2/routes/developer-profiles.ts b/src/services/extensions/v2/routes/developer-profiles.ts index 5c70b874..4a76dbad 100644 --- a/src/services/extensions/v2/routes/developer-profiles.ts +++ b/src/services/extensions/v2/routes/developer-profiles.ts @@ -387,7 +387,7 @@ export function registerDeveloperProfileRoutes(app: ExtensionsV2App): void { if ( !own.error && own.data && - own.data.id.toLowerCase() === id.toLowerCase() + own.data.identifier.toLowerCase() === id.toLowerCase() ) { // The full owned view carries contact_email, verification signals, // and transfer state, so it stays behind the active-account check @@ -411,7 +411,7 @@ export function registerDeveloperProfileRoutes(app: ExtensionsV2App): void { return c.json(errorBody(access.error, "Unable to check access"), 500); } if (access.data?.moderator) { - const { data, error } = await db.getById(id); + const { data, error } = await db.getByIdentifier(id); if (error || !data) { const status = statusFromErrorCode(error?.code, false); return c.json(errorBody(error, "Developer not found"), status); @@ -421,7 +421,7 @@ export function registerDeveloperProfileRoutes(app: ExtensionsV2App): void { return res; } } - const { data, error } = await db.getById(id); + const { data, error } = await db.getByIdentifier(id); if (error || !data) { const status = statusFromErrorCode(error?.code, false); return c.json(errorBody(error, "Developer not found"), status); diff --git a/src/services/extensions/v2/routes/moderation.ts b/src/services/extensions/v2/routes/moderation.ts index 85f9fac1..5cd12a3b 100644 --- a/src/services/extensions/v2/routes/moderation.ts +++ b/src/services/extensions/v2/routes/moderation.ts @@ -217,7 +217,8 @@ export function registerModerationRoutes(app: ExtensionsV2App): void { "application/json": { schema: z.object({ result: z.object({ - id: z.string(), + id: z.uuidv7(), + identifier: z.string(), status: z.literal("delisted"), notified: NotifiedSchema }) @@ -256,7 +257,14 @@ export function registerModerationRoutes(app: ExtensionsV2App): void { reason }); return c.json( - { result: { id: data.id, status: "delisted" as const, notified } }, + { + result: { + id: data.id, + identifier: data.identifier, + status: "delisted" as const, + notified + } + }, 200 ); }); @@ -281,7 +289,8 @@ export function registerModerationRoutes(app: ExtensionsV2App): void { "application/json": { schema: z.object({ result: z.object({ - id: z.string(), + id: z.uuidv7(), + identifier: z.string(), status: z.literal("relisted"), notified: NotifiedSchema }) @@ -315,11 +324,18 @@ export function registerModerationRoutes(app: ExtensionsV2App): void { } const notified = await notifyAuthor(c, extDb, query, { kind: "extension-relisted", - extensionId: data.id, + extensionId: id, reason: review_note?.trim() || undefined }); return c.json( - { result: { id: data.id, status: "relisted" as const, notified } }, + { + result: { + id: data.id, + identifier: data.identifier, + status: "relisted" as const, + notified + } + }, 200 ); }); @@ -356,7 +372,8 @@ export function registerModerationRoutes(app: ExtensionsV2App): void { "application/json": { schema: z.object({ result: z.object({ - id: z.string(), + id: z.uuidv7(), + identifier: z.string(), revision_id: z.string(), status: z.literal("approved") }) @@ -407,6 +424,7 @@ export function registerModerationRoutes(app: ExtensionsV2App): void { { result: { id: data.id, + identifier: data.identifier, revision_id: data.revisionId, status: "approved" as const } @@ -435,7 +453,8 @@ export function registerModerationRoutes(app: ExtensionsV2App): void { "application/json": { schema: z.object({ result: z.object({ - id: z.string(), + id: z.uuidv7(), + identifier: z.string(), approved: z.literal(true), notified: NotifiedSchema }) @@ -473,7 +492,7 @@ export function registerModerationRoutes(app: ExtensionsV2App): void { } const notified = await notifyAuthor(c, extDb, query, { kind: "developer-approved", - developerId: id + developerId: data.id }); return c.json({ result: { ...data, notified } }, 200); }); diff --git a/src/services/extensions/v2/routes/owner-extensions.ts b/src/services/extensions/v2/routes/owner-extensions.ts index 3421e00a..53768b9b 100644 --- a/src/services/extensions/v2/routes/owner-extensions.ts +++ b/src/services/extensions/v2/routes/owner-extensions.ts @@ -72,12 +72,13 @@ async function revisionReadAccess( return { data: { extensionId: ownership.data.extensionId }, error: null }; } +const AcceptedRevisionResultSchema = z.object({ + identifier: z.string(), + revision_id: z.string(), + status: z.literal("pending") +}); const AcceptedRevisionSchema = z.object({ - result: z.object({ - id: z.string(), - revision_id: z.string(), - status: z.literal("pending") - }) + result: AcceptedRevisionResultSchema }); export function registerOwnerExtensionsRoutes(app: ExtensionsV2App): void { @@ -95,7 +96,15 @@ export function registerOwnerExtensionsRoutes(app: ExtensionsV2App): void { }, responses: { 201: { - content: { "application/json": { schema: AcceptedRevisionSchema } }, + content: { + "application/json": { + schema: z.object({ + result: AcceptedRevisionResultSchema.extend({ + id: z.uuidv7() + }) + }) + } + }, description: "Extension created. It holds the id immediately but stays out of the public extension listing until a moderator approves the revision." }, @@ -120,14 +129,14 @@ export function registerOwnerExtensionsRoutes(app: ExtensionsV2App): void { app.openapi(createRouteDefinition, async (c) => { const auth = getAuth(c); - const { id, ...content } = c.req.valid("json"); + const { identifier, ...content } = c.req.valid("json"); const publisher = getPublisher(c); // A publisher client writes under its mapped developer; the mapping is // re-verified inside the insert. Humans resolve their own profile. if (publisher) { const input: CreateExtensionAsPublisherInput = { - extensionId: id, + identifier, clientId: publisher.clientId, developerId: publisher.developerId, submittedBy: auth.userId, @@ -146,6 +155,7 @@ export function registerOwnerExtensionsRoutes(app: ExtensionsV2App): void { { result: { id: data.id, + identifier, revision_id: data.revisionId, status: "pending" as const } @@ -175,7 +185,7 @@ export function registerOwnerExtensionsRoutes(app: ExtensionsV2App): void { const db = new ExtensionsDatabase(getExtensionsDb(c.env.DB_EXTENSIONS)); const { data, error } = await db.create({ - extensionId: id, + identifier, developerId: owner.data.id, ownershipEpoch: owner.data.ownershipEpoch, submittedBy: auth.userId, @@ -192,6 +202,7 @@ export function registerOwnerExtensionsRoutes(app: ExtensionsV2App): void { { result: { id: data.id, + identifier, revision_id: data.revisionId, status: "pending" as const } @@ -268,7 +279,13 @@ export function registerOwnerExtensionsRoutes(app: ExtensionsV2App): void { ); } return c.json( - { result: { id, revision_id: data.id, status: "pending" as const } }, + { + result: { + identifier: id, + revision_id: data.id, + status: "pending" as const + } + }, 202 ); }); @@ -337,7 +354,7 @@ export function registerOwnerExtensionsRoutes(app: ExtensionsV2App): void { return c.json( { result: { - id: data.extensionId, + identifier: data.identifier, revision_id: data.id, status: "pending" as const } @@ -359,7 +376,11 @@ export function registerOwnerExtensionsRoutes(app: ExtensionsV2App): void { content: { "application/json": { schema: z.object({ - result: z.object({ id: z.string(), deleted: z.literal(true) }) + result: z.object({ + id: z.uuidv7(), + identifier: z.string(), + deleted: z.literal(true) + }) }) } }, @@ -389,7 +410,16 @@ export function registerOwnerExtensionsRoutes(app: ExtensionsV2App): void { ); } revalidateExtensions(c); - return c.json({ result: { id: data.id, deleted: true as const } }, 200); + return c.json( + { + result: { + id: data.id, + identifier: data.identifier, + deleted: true as const + } + }, + 200 + ); }); const revisionsRoute = createRoute({ @@ -445,7 +475,7 @@ export function registerOwnerExtensionsRoutes(app: ExtensionsV2App): void { getExtensionsDb(c.env.DB_EXTENSIONS) ); const { data, error } = await db.listScoped({ - extensionId: access.data.extensionId, + extensionId: id, readerId: auth.userId, sort: "newest", limit, @@ -514,11 +544,7 @@ export function registerOwnerExtensionsRoutes(app: ExtensionsV2App): void { const db = new ExtensionRevisionsDatabase( getExtensionsDb(c.env.DB_EXTENSIONS) ); - const { data, error } = await db.getById( - access.data.extensionId, - revisionId, - auth.userId - ); + const { data, error } = await db.getById(id, revisionId, auth.userId); if (error || !data) { return c.json( errorBody(error, "Unable to load revisions"), diff --git a/src/services/extensions/v2/routes/packages.ts b/src/services/extensions/v2/routes/packages.ts index 7e6e093c..0b97cca7 100644 --- a/src/services/extensions/v2/routes/packages.ts +++ b/src/services/extensions/v2/routes/packages.ts @@ -75,7 +75,7 @@ async function jobResponse(db: D1Database, job: PackageJob) { return { result: { id: job.id, - extension_id: job.manifest.id, + extension_identifier: job.manifest.id, status: job.status, artifact: artifact ? artifactResponse(artifact).result : null, error: job.status === "failed" ? job.error : null, @@ -200,7 +200,7 @@ export function registerPackageRoutes(app: ExtensionsV2App) { SELECT d.owner_user_id FROM package_sources s JOIN developers d ON d.profile_generation=s.profile_generation JOIN users moderator ON moderator.id=? AND moderator.is_moderator=1 AND moderator.deleted_at IS NULL - WHERE s.id=? AND s.extension_id=? AND s.version=? + WHERE s.id=? AND s.extension_identifier=? AND s.version=? AND EXISTS (SELECT 1 FROM package_artifacts a WHERE a.source_id=s.id AND a.published_at IS NOT NULL) ` ) @@ -436,7 +436,7 @@ export function registerPackageRoutes(app: ExtensionsV2App) { const auth = getOptionalAuth(c); const row = await c.env.DB_EXTENSIONS.prepare( `${ARTIFACT_SELECT} WHERE a.id=? AND ( - (a.published_at IS NOT NULL AND EXISTS (SELECT 1 FROM extensions e WHERE lower(e.id)=lower(s.extension_id) AND e.published_at IS NOT NULL AND e.delisted_at IS NULL)) + (a.published_at IS NOT NULL AND EXISTS (SELECT 1 FROM extensions e WHERE lower(e.identifier)=lower(s.extension_identifier) AND e.published_at IS NOT NULL AND e.delisted_at IS NULL)) OR EXISTS (SELECT 1 FROM users u WHERE u.id=? AND u.deleted_at IS NULL AND (u.is_moderator=1 OR EXISTS (SELECT 1 FROM developers d WHERE d.profile_generation=s.profile_generation AND d.owner_user_id=u.id))) )` ) @@ -446,7 +446,8 @@ export function registerPackageRoutes(app: ExtensionsV2App) { throw new PackageError("NOT_FOUND", "Package not found", 404); if (!download && !icon) return c.json(artifactResponse(row), 200); if (icon) { - if (row.published_at) scheduleIconPublication(c, row.extension_id); + if (row.published_at) + scheduleIconPublication(c, row.extension_identifier); const avif = (c.req.header("Accept") ?? "") .split(",") .some((part) => { @@ -509,7 +510,7 @@ export function registerPackageRoutes(app: ExtensionsV2App) { return new Response(body, { headers: { "Content-Type": "application/zip", - "Content-Disposition": `attachment; filename="${row.extension_id}-${row.version}-${row.id}.zip"`, + "Content-Disposition": `attachment; filename="${row.extension_identifier}-${row.version}-${row.id}.zip"`, ETag: `"${row.sha256}"`, "X-Content-Type-Options": "nosniff", "Cache-Control": "private, no-store", diff --git a/src/services/extensions/v2/routes/public-extensions.ts b/src/services/extensions/v2/routes/public-extensions.ts index 39796fdb..ca36e1d4 100644 --- a/src/services/extensions/v2/routes/public-extensions.ts +++ b/src/services/extensions/v2/routes/public-extensions.ts @@ -48,18 +48,18 @@ export function registerPublicExtensionsRoutes(app: ExtensionsV2App): void { }); app.openapi(listExtensionsRoute, async (c) => { - const { scope, type, developer_id, status, q, limit, cursor } = + const { scope, type, developer_identifier, status, q, limit, cursor } = c.req.valid("query"); const auth = getOptionalAuth(c); const extDb = getExtensionsDb(c.env.DB_EXTENSIONS); // Filter/scope combinations are rejected rather than silently ignored so // a caller cannot mistake one projection for another. - if (scope !== "public" && developer_id !== undefined) { + if (scope !== "public" && developer_identifier !== undefined) { return c.json( { error: { - message: "developer_id is only valid with scope=public", + message: "developer_identifier is only valid with scope=public", code: "VALIDATION_ERROR" } }, @@ -82,7 +82,7 @@ export function registerPublicExtensionsRoutes(app: ExtensionsV2App): void { const db = new ExtensionsDatabase(extDb); const { data, error } = await db.list({ type, - developerId: developer_id, + developerIdentifier: developer_identifier, limit, cursor }); @@ -322,7 +322,7 @@ export function registerPublicExtensionsRoutes(app: ExtensionsV2App): void { } } - const { data, error } = await db.getById(id); + const { data, error } = await db.getByIdentifier(id); if (error || !data) { const status = statusFromContentReadError(error?.code); return c.json(errorBody(error, "Extension not found"), status); diff --git a/src/services/extensions/v2/schemas/developers.ts b/src/services/extensions/v2/schemas/developers.ts index 413d9d47..bd4af04a 100644 --- a/src/services/extensions/v2/schemas/developers.ts +++ b/src/services/extensions/v2/schemas/developers.ts @@ -20,7 +20,7 @@ const developerId = () => export const DeveloperSchema = z .object({ - id: developerId(), + identifier: developerId(), type: z.enum(["user", "organization"]), name: z.string().min(1).max(120), URL: httpUrl().optional(), @@ -43,6 +43,7 @@ export const DeveloperInputSchema = DeveloperSchema.strict().openapi("DeveloperInput"); export const DeveloperProfileSchema = DeveloperSchema.extend({ + id: z.uuidv7(), approved: z.boolean(), content_revision: z.int().positive(), profile_generation: ProfileGenerationSchema, @@ -107,6 +108,7 @@ export function toPublicDeveloper( ): PublicDeveloper { return { id: profile.id, + identifier: profile.identifier, type: profile.type, name: profile.name, URL: profile.URL, diff --git a/src/services/extensions/v2/schemas/extensions.ts b/src/services/extensions/v2/schemas/extensions.ts index b678d480..9b902ecd 100644 --- a/src/services/extensions/v2/schemas/extensions.ts +++ b/src/services/extensions/v2/schemas/extensions.ts @@ -132,16 +132,16 @@ function refineContentSize(content: unknown, ctx: z.RefinementCtx): void { } } -// POST /extensions. The id is chosen once here and is immutable afterwards. +// POST /extensions. The public identifier is chosen once; the API generates its UUID. // No developer field: a user owns at most one profile, so the server knows it. export const ExtensionCreateSchema = ExtensionContentSchema.extend({ - id: ManifestSchema.shape.id + identifier: ManifestSchema.shape.id }) .strict() .superRefine(refineContentSize) .openapi("ExtensionCreate"); -// PUT /extensions/{id}. Same content, no id — that comes from the path. +// PUT /extensions/{id}. The public identifier comes from the path. // .strict() belongs here and not on ExtensionContentSchema, which is also a // branch of the responses below; see DeveloperInputSchema for why. export const ExtensionUpdateSchema = ExtensionContentSchema.strict() @@ -162,7 +162,8 @@ const PublishedContentSchema = ExtensionContentSchema.extend({ // The public projection: published package attribution plus its current owner. export const ExtensionSchema = PublishedContentSchema.extend({ - id: z.string(), + id: z.uuidv7(), + identifier: z.string(), developer: PublicDeveloperSchema }).openapi("Extension"); @@ -190,7 +191,8 @@ const ExtensionCardContentSchema = ExtensionContentSchema.omit({ }); export const ExtensionListItemSchema = ExtensionCardContentSchema.extend({ - id: z.string(), + id: z.uuidv7(), + identifier: z.string(), developer: PublicDeveloperSchema }).openapi("ExtensionListItem"); @@ -243,7 +245,8 @@ export const DelistedInfoSchema = z // no derived `status` field on top; see the README for how they map to a UI. export const OwnedExtensionListItemSchema = z .object({ - id: z.string(), + id: z.uuidv7(), + identifier: z.string(), developer: PublicDeveloperSchema, published: ExtensionCardContentSchema.nullable(), pending_revision: PendingRevisionRefSchema.nullable(), @@ -276,11 +279,11 @@ export const ExtensionListQuerySchema = z.object({ .openapi({ param: { name: "type", in: "query" } }), - developer_id: z + developer_identifier: z .string() .optional() .openapi({ - param: { name: "developer_id", in: "query" } + param: { name: "developer_identifier", in: "query" } }), ...cursorPaginationFields }); @@ -305,11 +308,11 @@ export const UnifiedExtensionListQuerySchema = ExtensionListQuerySchema.extend({ description: "public: published extension directory (anonymous allowed). mine: caller's own extensions. all: every extension (moderator only)." }), - developer_id: z + developer_identifier: z .string() .optional() .openapi({ - param: { name: "developer_id", in: "query" }, + param: { name: "developer_identifier", in: "query" }, description: "Public scope only" }), status: ExtensionStatusSchema.optional().openapi({ diff --git a/src/services/extensions/v2/schemas/m2m.ts b/src/services/extensions/v2/schemas/m2m.ts index ba3266eb..1b9d8325 100644 --- a/src/services/extensions/v2/schemas/m2m.ts +++ b/src/services/extensions/v2/schemas/m2m.ts @@ -19,7 +19,7 @@ export const PublisherClientIdParamSchema = z.object({ // developer's pending revisions); without it the client may only propose. export const PublisherRegisterSchema = z .object({ - developer_id: z.string().min(1).max(200), + developer_id: z.uuidv7(), can_publish: z.boolean().default(false) }) .strict() @@ -28,7 +28,7 @@ export const PublisherRegisterSchema = z export const PublisherClientSchema = z .object({ client_id: z.string(), - developer_id: z.string(), + developer_id: z.uuidv7(), can_publish: z.boolean(), created_at: z.string(), revoked_at: z.string().nullable() diff --git a/src/services/extensions/v2/schemas/ownership.ts b/src/services/extensions/v2/schemas/ownership.ts index 0d49efcb..0722c88b 100644 --- a/src/services/extensions/v2/schemas/ownership.ts +++ b/src/services/extensions/v2/schemas/ownership.ts @@ -18,7 +18,7 @@ export type DeveloperTransfer = z.infer; export const DeveloperClaimSchema = z .object({ id: z.string(), - developer_id: z.string(), + developer_id: z.uuidv7(), claimant_id: z.string(), status: z.enum(["pending", "approved", "rejected"]), note: z.string().optional(), @@ -39,6 +39,7 @@ export const DeveloperClaimSchema = z export type DeveloperClaim = z.infer; export const PendingDeveloperClaimSchema = DeveloperClaimSchema.extend({ + developer_identifier: z.string(), developer_name: z.string(), developer_type: z.enum(["user", "organization"]), // The claimant's own account name/GitHub handle, so the moderator sees diff --git a/src/services/extensions/v2/schemas/packages.ts b/src/services/extensions/v2/schemas/packages.ts index cd34e47d..64f90d40 100644 --- a/src/services/extensions/v2/schemas/packages.ts +++ b/src/services/extensions/v2/schemas/packages.ts @@ -36,7 +36,7 @@ export const PackageJobParamSchema = z.object({ jobId: z.uuid() }); export const PackageJobSchema = z .object({ id: z.uuid(), - extension_id: z.string(), + extension_identifier: z.string(), status: z.enum(["queued", "processing", "ready", "failed"]), artifact: z.union([PackageArtifactSchema, z.null()]), error: z diff --git a/src/services/extensions/v2/schemas/revisions.ts b/src/services/extensions/v2/schemas/revisions.ts index 99f03b3d..6db200d5 100644 --- a/src/services/extensions/v2/schemas/revisions.ts +++ b/src/services/extensions/v2/schemas/revisions.ts @@ -11,7 +11,8 @@ export type RevisionStatus = z.infer; export const ExtensionRevisionSchema = z .object({ id: z.string(), - extension_id: z.string(), + extension_id: z.uuidv7(), + extension_identifier: z.string(), developer_id: z.string(), submitted_by: z.string(), status: RevisionStatusSchema, diff --git a/test/services/extensions/v1/index.test.ts b/test/services/extensions/v1/index.test.ts index 3dff1c47..41f6adbc 100644 --- a/test/services/extensions/v1/index.test.ts +++ b/test/services/extensions/v1/index.test.ts @@ -1,3 +1,4 @@ +import { v7 as uuidv7 } from "uuid"; import { describe, it, expect, beforeAll, beforeEach } from "vitest"; import { eq } from "drizzle-orm"; import { @@ -15,7 +16,8 @@ import { applyTestMigrations } from "../../../utils/apply-migrations"; const testExtensionRows = [ { - id: "Example", + id: uuidv7(), + identifier: "Example", manifest: JSON.stringify({ manifest_version: 1, author: { name: "FOSSBilling" }, @@ -59,7 +61,8 @@ const testExtensionRows = [ "https://github.com/FOSSBilling/example-module/releases/download/0.0.5/Example.zip" }, { - id: "TestTheme", + id: uuidv7(), + identifier: "TestTheme", manifest: JSON.stringify({ manifest_version: 1, author: { name: "FOSSBilling" }, @@ -104,6 +107,7 @@ describe("Extensions API v1", () => { await db.insert(developers).values({ id: "fossbilling", + identifier: "fossbilling", type: "organization", name: "fossbilling", url: "https://fossbilling.org" @@ -199,7 +203,8 @@ describe("Extensions API v1", () => { const db = getExtensionsDb(env.DB_EXTENSIONS); await db.insert(extensions).values({ ...testExtensionRows[0], - id: "alpha-mod", + id: uuidv7(), + identifier: "alpha-mod", name: "Alpha Module", publishedAt: "2026-01-01T00:00:00.000Z" }); @@ -275,7 +280,7 @@ describe("Extensions API v1", () => { await db .update(extensions) .set({ delistedAt: "2026-01-01T00:00:00.000Z" }) - .where(eq(extensions.id, "Example")); + .where(eq(extensions.identifier, "Example")); const ctx = createExecutionContext(); const res = await app.request("/extensions/v1/list", {}, env, ctx); @@ -309,13 +314,15 @@ describe("Extensions API v1", () => { const db = getExtensionsDb(env.DB_EXTENSIONS); await db.insert(developers).values({ id: "nourl", + identifier: "nourl", type: "user", name: "No URL", url: null }); await db.insert(extensions).values({ ...testExtensionRows[1], - id: "NoIcon", + id: uuidv7(), + identifier: "NoIcon", developerId: "nourl", iconUrl: null, publishedAt: "2026-01-01T00:00:00.000Z" @@ -382,7 +389,7 @@ describe("Extensions API v1", () => { await db .update(extensions) .set({ delistedAt: "2026-01-01T00:00:00.000Z" }) - .where(eq(extensions.id, "Example")); + .where(eq(extensions.identifier, "Example")); const ctx = createExecutionContext(); const res = await app.request("/extensions/v1/Example", {}, env, ctx); @@ -433,7 +440,7 @@ describe("Extensions API v1", () => { await db .update(extensions) .set({ releases: JSON.stringify(releases), version: expected }) - .where(eq(extensions.id, "Example")); + .where(eq(extensions.identifier, "Example")); const ctx = createExecutionContext(); const response = await app.request( "/extensions/v1/Example/version", @@ -509,7 +516,7 @@ describe("Extensions API v1", () => { } ]) }) - .where(eq(extensions.id, "Example")); + .where(eq(extensions.identifier, "Example")); const ctx = createExecutionContext(); const response = await app.request( "/extensions/v1/Example/version", @@ -533,6 +540,7 @@ describe("v1 projection failures", () => { await db.delete(developers); await db.insert(developers).values({ id: "projection-owner", + identifier: "projection-owner", type: "organization", name: "Owner" }); diff --git a/test/services/extensions/v2/account.test.ts b/test/services/extensions/v2/account.test.ts index 8adb66d9..749359c8 100644 --- a/test/services/extensions/v2/account.test.ts +++ b/test/services/extensions/v2/account.test.ts @@ -1,3 +1,4 @@ +import { UUID_V7 } from "./db-fixtures"; import app from "../../../../src/app"; import { env } from "cloudflare:workers"; import { @@ -24,6 +25,7 @@ import { } from "./harness"; import { insertUser, + getDeveloper, insertDeveloper, insertExtension, insertUnpublishedExtension, @@ -241,15 +243,15 @@ describe("Extensions API v2", () => { expect(await developer.json()).toEqual({ result: null }); await insertDeveloper(db, { - id: "account-developer", + identifier: "account-developer", type: "user", name: "Account Developer", owner_user_id: "account-1" }); await insertExtension(db, { - id: "account-extension", + identifier: "account-extension", type: "module", - developer_id: "account-developer", + developer_identifier: "account-developer", name: "Account Extension", description: "description", releases: "[]", @@ -265,12 +267,17 @@ describe("Extensions API v2", () => { const owned = await get("/extensions/v2/extensions?scope=mine", headers); expect(owned.status).toBe(200); expect(await owned.json()).toMatchObject({ - result: [{ id: "account-extension" }], + result: [ + { + id: expect.stringMatching(UUID_V7), + identifier: "account-extension" + } + ], pagination: { has_more: false, next_cursor: null } }); const filtered = await get( - "/extensions/v2/extensions?scope=mine&developer_id=someone-else", + "/extensions/v2/extensions?scope=mine&developer_identifier=someone-else", headers ); expect(filtered.status).toBe(422); @@ -422,7 +429,10 @@ describe("Extensions API v2", () => { .prepare( "INSERT INTO m2m_publisher_clients (client_id, developer_id, can_publish) VALUES (?, ?, 1)" ) - .bind("modules-publisher", "mapped-developer") + .bind( + "modules-publisher", + (await getDeveloper(db, "mapped-developer"))!.id + ) .run(); const before = await db .prepare("SELECT developer_id FROM m2m_publisher_clients") @@ -456,13 +466,13 @@ describe("Extensions API v2", () => { it("blocks deletion while an unpublished extension is still owned", async () => { await seedDeveloper("pending-developer", "pending-owner"); await insertUnpublishedExtension(db, { - id: "pending-ext", - developer_id: "pending-developer" + identifier: "pending-ext", + developer_identifier: "pending-developer" }); await insertRevision(db, { id: "pending-revision", - extension_id: "pending-ext", - developer_id: "pending-developer", + extension_identifier: "pending-ext", + developer_identifier: "pending-developer", submitted_by: "pending-owner", content: JSON.stringify(sampleContent()) }); @@ -487,37 +497,37 @@ describe("Extensions API v2", () => { await seedUnownedDeveloper("claim-target"); await insertDeveloperTransfer(db, { id: "cleanup-transfer", - developer_id: "cleanup-developer", + developer_identifier: "cleanup-developer", token_hash: "cleanup-token-hash", created_by: "cleanup-user", expires_at: "2099-01-01 00:00:00" }); await insertDeveloperClaim(db, { id: "cleanup-owned-claim", - developer_id: "cleanup-developer", + developer_identifier: "cleanup-developer", claimant_id: "cleanup-user" }); await insertDeveloperClaim(db, { id: "cleanup-pending-claim", - developer_id: "claim-target", + developer_identifier: "claim-target", claimant_id: "cleanup-user" }); // Under claim-target, a developer this user does not own - which is the // only way a pending revision survives the "no owned extensions" guard. await insertUnpublishedExtension(db, { - id: "cleanup-pending-ext", - developer_id: "claim-target" + identifier: "cleanup-pending-ext", + developer_identifier: "claim-target" }); await insertRevision(db, { id: "cleanup-pending-revision", - extension_id: "cleanup-pending-ext", - developer_id: "claim-target", + extension_identifier: "cleanup-pending-ext", + developer_identifier: "claim-target", submitted_by: "cleanup-user", content: JSON.stringify(sampleContent()) }); await insertDeveloperHistory(db, { id: "cleanup-history", - developer_id: "cleanup-developer", + developer_identifier: "cleanup-developer", type: "user", name: "Before deletion", changed_by: "cleanup-user" @@ -569,7 +579,7 @@ describe("Extensions API v2", () => { expect(await listDeveloperHistory(db)).toEqual([ expect.objectContaining({ id: "cleanup-history", - developer_id: "cleanup-developer", + developer_id: expect.stringMatching(UUID_V7), changed_by: "cleanup-user" }) ]); @@ -601,7 +611,7 @@ describe("Extensions API v2", () => { await seedDeveloper("rollback-developer", "rollback-user"); await insertDeveloperTransfer(db, { id: "rollback-transfer", - developer_id: "rollback-developer", + developer_identifier: "rollback-developer", token_hash: "rollback-token-hash", created_by: "rollback-user", expires_at: "2099-01-01 00:00:00" diff --git a/test/services/extensions/v2/badges.test.ts b/test/services/extensions/v2/badges.test.ts index 9a3a3811..4995ca4c 100644 --- a/test/services/extensions/v2/badges.test.ts +++ b/test/services/extensions/v2/badges.test.ts @@ -21,13 +21,13 @@ const badge = (type: string) => beforeEach(async () => { await insertDeveloper(db, { - id: "badge-dev", + identifier: "badge-dev", type: "user", name: "Developer" }); await insertExtension(db, { - id: "example", - developer_id: "badge-dev", + identifier: "example", + developer_identifier: "badge-dev", license: '"MIT OR Apache-2.0"', // Deliberately unordered; requirements belong to each release, not to the // root manifest or to the first element of a stored array. @@ -46,7 +46,7 @@ beforeEach(async () => { describe("v2 public badges", () => { it("uses the selected release for badges and the legacy version endpoint", async () => { await db - .prepare("UPDATE extensions SET releases=? WHERE id='example'") + .prepare("UPDATE extensions SET releases=? WHERE identifier='example'") .bind( JSON.stringify([ { ...release("1.0.0", "^0.7"), is_latest: true }, @@ -67,7 +67,7 @@ describe("v2 public badges", () => { it("prefers stable releases over newer prereleases when no release is selected", async () => { await db - .prepare("UPDATE extensions SET releases=? WHERE id='example'") + .prepare("UPDATE extensions SET releases=? WHERE identifier='example'") .bind( JSON.stringify([ release("3.0.0-beta.1", "^1.0"), @@ -130,15 +130,19 @@ describe("v2 public badges", () => { "hides %s extensions", async (kind) => { if (kind === "missing") - await db.prepare("DELETE FROM extensions WHERE id='example'").run(); + await db + .prepare("DELETE FROM extensions WHERE identifier='example'") + .run(); if (kind === "unpublished") await db - .prepare("UPDATE extensions SET published_at=NULL WHERE id='example'") + .prepare( + "UPDATE extensions SET published_at=NULL WHERE identifier='example'" + ) .run(); if (kind === "delisted") await db .prepare( - "UPDATE extensions SET delisted_at=CURRENT_TIMESTAMP, delist_reason='Removed' WHERE id='example'" + "UPDATE extensions SET delisted_at=CURRENT_TIMESTAMP, delist_reason='Removed' WHERE identifier='example'" ) .run(); const response = await badge("fossbilling"); @@ -158,7 +162,7 @@ describe("v2 public badges", () => { it("handles empty release histories explicitly", async () => { await db - .prepare("UPDATE extensions SET releases='[]' WHERE id='example'") + .prepare("UPDATE extensions SET releases='[]' WHERE identifier='example'") .run(); expect(await (await badge("version")).text()).toContain( "Latest version: unknown" @@ -173,7 +177,7 @@ describe("v2 public badges", () => { it("escapes requirement text in the generated SVG", async () => { await db - .prepare("UPDATE extensions SET releases=? WHERE id='example'") + .prepare("UPDATE extensions SET releases=? WHERE identifier='example'") .bind( JSON.stringify([release("1.0.0", '')]) ) @@ -185,7 +189,7 @@ describe("v2 public badges", () => { it("renders badges when README content exceeds detail limits", async () => { await db - .prepare("UPDATE extensions SET readme=? WHERE id='example'") + .prepare("UPDATE extensions SET readme=? WHERE identifier='example'") .bind("x".repeat(300_000)) .run(); const queries: string[] = []; @@ -198,7 +202,7 @@ describe("v2 public badges", () => { it("refuses oversized stored badge fields without truncation", async () => { await db - .prepare("UPDATE extensions SET license=? WHERE id='example'") + .prepare("UPDATE extensions SET license=? WHERE identifier='example'") .bind(JSON.stringify("A".repeat(1001))) .run(); const response = await badge("license"); diff --git a/test/services/extensions/v2/contract.test.ts b/test/services/extensions/v2/contract.test.ts index b031b82c..b66ea1b9 100644 --- a/test/services/extensions/v2/contract.test.ts +++ b/test/services/extensions/v2/contract.test.ts @@ -1,3 +1,4 @@ +import { UUID_V7 } from "./db-fixtures"; import { describe, it, expect, vi } from "vitest"; import { setupExtensionsV2Tests, @@ -36,7 +37,10 @@ describe("Extensions API v2 contract", () => { ); expect(me.status).toBe(200); await expect(me.json()).resolves.toMatchObject({ - result: { id: "dev-developer" } + result: { + id: expect.stringMatching(UUID_V7), + identifier: "dev-developer" + } }); }); @@ -57,7 +61,7 @@ describe("Extensions API v2 contract", () => { // developer_id is public-scope only. const scoped = await get( - "/extensions/v2/extensions?scope=mine&developer_id=dev-developer", + "/extensions/v2/extensions?scope=mine&developer_identifier=dev-developer", mod ); expect(scoped.status).toBe(422); @@ -89,14 +93,14 @@ describe("Extensions API v2 contract", () => { it("walks past the first page when rows exceed the limit", async () => { await insertDeveloper(db, { - id: "aaa-developer", + identifier: "aaa-developer", type: "user", name: "Aaa Developer", url: null, owner_user_id: "user-1" }); await insertDeveloper(db, { - id: "zzz-developer", + identifier: "zzz-developer", type: "user", name: "Zzz Developer", url: null, @@ -108,10 +112,12 @@ describe("Extensions API v2 contract", () => { const first = await get("/extensions/v2/developers?limit=1", mod); expect(first.status).toBe(200); const firstBody = (await first.json()) as { - result: Array<{ id: string }>; + result: Array<{ id: string; identifier: string }>; pagination: { next_cursor: string | null; has_more: boolean }; }; - expect(firstBody.result.map((d) => d.id)).toEqual(["aaa-developer"]); + expect(firstBody.result.map((d) => d.identifier)).toEqual([ + "aaa-developer" + ]); expect(firstBody.pagination.has_more).toBe(true); expect(firstBody.pagination.next_cursor).toBeTruthy(); @@ -121,10 +127,12 @@ describe("Extensions API v2 contract", () => { ); expect(second.status).toBe(200); const secondBody = (await second.json()) as { - result: Array<{ id: string }>; + result: Array<{ id: string; identifier: string }>; pagination: { next_cursor: string | null; has_more: boolean }; }; - expect(secondBody.result.map((d) => d.id)).toEqual(["zzz-developer"]); + expect(secondBody.result.map((d) => d.identifier)).toEqual([ + "zzz-developer" + ]); expect(secondBody.pagination).toEqual({ next_cursor: null, has_more: false diff --git a/test/services/extensions/v2/db-fixtures.ts b/test/services/extensions/v2/db-fixtures.ts index 2dc871ab..9cf3beae 100644 --- a/test/services/extensions/v2/db-fixtures.ts +++ b/test/services/extensions/v2/db-fixtures.ts @@ -1,3 +1,7 @@ +import { v7 as uuidv7 } from "uuid"; + +export const UUID_V7 = + /^[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/; // Raw-SQL test fixtures against the real local D1 (see // test/utils/apply-migrations.ts) - deliberately not going through Drizzle, // since these exist to seed/read rows for assertions, not to exercise the @@ -6,6 +10,7 @@ export interface DeveloperRow { id: string; + identifier: string; type: string; name: string; url: string | null; @@ -28,6 +33,7 @@ export interface DeveloperRow { export interface ExtensionRow { id: string; + identifier: string; developer_id: string; published_at: string | null; published_revision_id: string | null; @@ -231,7 +237,11 @@ export async function ensureUser(db: D1Database, id: string): Promise { export async function insertDeveloper( db: D1Database, - row: Partial & { id: string; type: string; name: string } + row: Omit, "id"> & { + identifier: string; + type: string; + name: string; + } ): Promise { if (row.owner_user_id) { await ensureUser(db, row.owner_user_id); @@ -240,14 +250,15 @@ export async function insertDeveloper( await db .prepare( `INSERT INTO developers - (id, type, name, url, owner_user_id, approved_at, created_at, updated_at, + (id, identifier, type, name, url, owner_user_id, approved_at, created_at, updated_at, avatar_url, contact_email, ownership_epoch, content_revision, approved_revision, approved_by, github_org_verified, github_verification_note, github_verified_at, github_url_verified) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` ) .bind( - row.id, + uuidv7(), + row.identifier, row.type, row.name, row.url ?? null, @@ -277,21 +288,25 @@ export async function insertDeveloper( // together, which is what extensions_published_content_check requires. export async function insertExtension( db: D1Database, - row: Partial & { id: string; developer_id: string } + row: Omit, "id" | "developer_id"> & { + identifier: string; + developer_identifier: string; + } ): Promise { const now = new Date().toISOString(); await db .prepare( `INSERT INTO extensions - (id, developer_id, published_at, published_revision_id, type, name, + (id, identifier, developer_id, published_at, published_revision_id, type, name, description, releases, website, license, icon_url, readme, source, version, download_url, created_at, updated_at, delisted_at, delist_reason, manifest) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` ) .bind( - row.id, - row.developer_id, + uuidv7(), + row.identifier, + (await getDeveloper(db, row.developer_identifier))!.id, row.published_at !== undefined ? row.published_at : now, row.published_revision_id ?? null, row.type ?? "module", @@ -323,24 +338,30 @@ export async function insertExtension( // published_at. This is the state a POST /extensions leaves behind. export async function insertUnpublishedExtension( db: D1Database, - row: { id: string; developer_id: string; created_at?: string } + row: { identifier: string; developer_identifier: string; created_at?: string } ): Promise { const now = new Date().toISOString(); await db .prepare( - `INSERT INTO extensions (id, developer_id, created_at, updated_at) - VALUES (?, ?, ?, ?)` + `INSERT INTO extensions (id, identifier, developer_id, created_at, updated_at) + VALUES (?, ?, ?, ?, ?)` + ) + .bind( + uuidv7(), + row.identifier, + (await getDeveloper(db, row.developer_identifier))!.id, + row.created_at ?? now, + now ) - .bind(row.id, row.developer_id, row.created_at ?? now, now) .run(); } export async function insertRevision( db: D1Database, - row: Partial & { + row: Omit, "developer_id" | "extension_id"> & { id: string; - extension_id: string; - developer_id: string; + extension_identifier: string; + developer_identifier: string; submitted_by: string; content: string; } @@ -358,8 +379,8 @@ export async function insertRevision( ) .bind( row.id, - row.extension_id, - row.developer_id, + (await getExtension(db, row.extension_identifier))!.id, + (await getDeveloper(db, row.developer_identifier))!.id, row.submitted_by, row.status ?? "pending", row.content, @@ -374,9 +395,9 @@ export async function insertRevision( export async function insertDeveloperClaim( db: D1Database, - row: Partial & { + row: Omit, "developer_id"> & { id: string; - developer_id: string; + developer_identifier: string; claimant_id: string; } ): Promise { @@ -393,7 +414,7 @@ export async function insertDeveloperClaim( ) .bind( row.id, - row.developer_id, + (await getDeveloper(db, row.developer_identifier))!.id, row.claimant_id, row.status ?? "pending", row.note ?? null, @@ -409,9 +430,9 @@ export async function insertDeveloperClaim( export async function insertDeveloperTransfer( db: D1Database, - row: Partial & { + row: Omit, "developer_id"> & { id: string; - developer_id: string; + developer_identifier: string; created_by: string; token_hash: string; expires_at: string; @@ -430,7 +451,7 @@ export async function insertDeveloperTransfer( ) .bind( row.id, - row.developer_id, + (await getDeveloper(db, row.developer_identifier))!.id, row.token_hash, row.created_by, row.created_at ?? new Date().toISOString(), @@ -444,9 +465,9 @@ export async function insertDeveloperTransfer( export async function insertDeveloperHistory( db: D1Database, - row: Partial & { + row: Omit, "developer_id"> & { id: string; - developer_id: string; + developer_identifier: string; type: string; name: string; changed_by: string; @@ -461,7 +482,7 @@ export async function insertDeveloperHistory( ) .bind( row.id, - row.developer_id, + (await getDeveloper(db, row.developer_identifier))!.id, row.type, row.name, row.url ?? null, @@ -473,11 +494,11 @@ export async function insertDeveloperHistory( export async function getDeveloper( db: D1Database, - id: string + identifier: string ): Promise { return db - .prepare("SELECT * FROM developers WHERE id = ?") - .bind(id) + .prepare("SELECT * FROM developers WHERE lower(identifier) = lower(?)") + .bind(identifier) .first(); } @@ -504,11 +525,11 @@ export async function countExtensions(db: D1Database): Promise { export async function getExtension( db: D1Database, - id: string + identifier: string ): Promise { return db - .prepare("SELECT * FROM extensions WHERE id = ?") - .bind(id) + .prepare("SELECT * FROM extensions WHERE lower(identifier) = lower(?)") + .bind(identifier) .first(); } @@ -604,7 +625,7 @@ export async function bumpDeveloperOwnership( await ensureUser(db, newOwnerUserId); await db .prepare( - "UPDATE developers SET owner_user_id = ?, ownership_epoch = ownership_epoch + 1 WHERE id = ?" + "UPDATE developers SET owner_user_id = ?, ownership_epoch = ownership_epoch + 1 WHERE identifier = ?" ) .bind(newOwnerUserId, developerId) .run(); diff --git a/test/services/extensions/v2/developer-approval.test.ts b/test/services/extensions/v2/developer-approval.test.ts index 1f29bcb0..988e3f9c 100644 --- a/test/services/extensions/v2/developer-approval.test.ts +++ b/test/services/extensions/v2/developer-approval.test.ts @@ -214,7 +214,7 @@ describe("Developer approval binds to the reviewed profile instance and content" ).toBe(200); // Model second-granular timestamps deterministically without timing a test. await db - .prepare("UPDATE developers SET updated_at = ? WHERE id = ?") + .prepare("UPDATE developers SET updated_at = ? WHERE identifier = ?") .bind("2026-10-07 00:00:00", "dev-developer") .run(); } @@ -246,7 +246,7 @@ describe("Developer approval binds to the reviewed profile instance and content" await insertUser(db, { id: "moderator", is_moderator: 1 }); const moderator = await authHeaders("moderator"); await insertDeveloper(db, { - id: "dev-developer", + identifier: "dev-developer", type: "user", name: "Developer", approved_at: "2026-10-07 00:00:00", @@ -254,8 +254,8 @@ describe("Developer approval binds to the reviewed profile instance and content" approved_revision: approvedRevision }); await insertExtension(db, { - id: "live-ext", - developer_id: "dev-developer" + identifier: "live-ext", + developer_identifier: "dev-developer" }); const expected = approvedRevision === 2; const profile = (await (await get(profilePath, {})).json()) as { @@ -273,10 +273,10 @@ describe("Developer approval binds to the reviewed profile instance and content" for (const query of ["scope", "status"]) { const queue = (await ( await get(`/extensions/v2/developers?${query}=unapproved`, moderator) - ).json()) as { result: Array<{ id: string }> }; - expect(queue.result.some((item) => item.id === "dev-developer")).toBe( - !expected - ); + ).json()) as { result: Array<{ id: string; identifier: string }> }; + expect( + queue.result.some((item) => item.identifier === "dev-developer") + ).toBe(!expected); } } ); diff --git a/test/services/extensions/v2/developer-profiles.test.ts b/test/services/extensions/v2/developer-profiles.test.ts index a9dbaa4d..7dda4758 100644 --- a/test/services/extensions/v2/developer-profiles.test.ts +++ b/test/services/extensions/v2/developer-profiles.test.ts @@ -1,3 +1,5 @@ +import { v7 as uuidv7 } from "uuid"; +import { UUID_V7 } from "./db-fixtures"; import { describe, it, expect, vi } from "vitest"; import { env } from "cloudflare:workers"; import { request as ghRequest } from "@octokit/request"; @@ -51,18 +53,18 @@ describe("Extensions API v2", () => { ).toBe(200); await db .prepare( - "UPDATE developers SET approved_at = CURRENT_TIMESTAMP, approved_revision = content_revision WHERE id = ?" + "UPDATE developers SET approved_at = CURRENT_TIMESTAMP, approved_revision = content_revision WHERE identifier = ?" ) - .bind(profile.id) + .bind(profile.identifier) .run(); - const before = await getDeveloper(db, profile.id); + const before = await getDeveloper(db, profile.identifier); const responses = await Promise.all( Array.from({ length: 6 }, () => put("/extensions/v2/developers/me", headers, profile) ) ); expect(responses.map((r) => r.status)).toEqual(Array(6).fill(200)); - expect(await getDeveloper(db, profile.id)).toEqual(before); + expect(await getDeveloper(db, profile.identifier)).toEqual(before); expect(await listDeveloperHistory(db)).toHaveLength(1); }); @@ -72,12 +74,13 @@ describe("Extensions API v2", () => { expect( (await put("/extensions/v2/developers/me", headers, profile)).status ).toBe(200); + const developerId = (await getDeveloper(db, profile.identifier))!.id; for (let i = 0; i < 18; i++) { await db .prepare( "INSERT INTO developer_history (id, developer_id, type, name, changed_by) VALUES (?, ?, 'user', 'Budget', ?)" ) - .bind(`budget-${i}`, profile.id, "budget-owner") + .bind(`budget-${i}`, developerId, "budget-owner") .run(); } const raced = await Promise.all( @@ -93,7 +96,7 @@ describe("Extensions API v2", () => { error: { code: "PROFILE_MUTATION_RATE_LIMITED" } }); expect(await listDeveloperHistory(db)).toHaveLength(20); - const current = await getDeveloper(db, profile.id); + const current = await getDeveloper(db, profile.identifier); expect( ( await put("/extensions/v2/developers/me", headers, { @@ -132,12 +135,13 @@ describe("Extensions API v2", () => { it("allows writes after old history leaves the window and audits all editable fields", async () => { const headers = await authHeaders("expired-budget-owner"); let profile = sampleDeveloper({ id: "expired-profile" }); + const formerDeveloperId = uuidv7(); for (let i = 0; i < 20; i++) { await db .prepare( "INSERT INTO developer_history (id, developer_id, type, name, changed_by, changed_at) VALUES (?, ?, 'user', 'Old', ?, datetime('now', '-2 days'))" ) - .bind(`old-${i}`, profile.id, "expired-budget-owner") + .bind(`old-${i}`, formerDeveloperId, "expired-budget-owner") .run(); } expect( @@ -156,13 +160,15 @@ describe("Extensions API v2", () => { ).toBe(200); } expect(await listDeveloperHistory(db)).toHaveLength(26); - expect((await getDeveloper(db, profile.id))?.content_revision).toBe(6); + expect( + (await getDeveloper(db, profile.identifier))?.content_revision + ).toBe(6); }); it("treats omitted optional fields as null and audits clearing them once", async () => { const headers = await authHeaders("null-profile-owner"); const profile = { - id: "null-profile", + identifier: "null-profile", type: "user", name: "Null profile" }; @@ -190,7 +196,7 @@ describe("Extensions API v2", () => { (await put("/extensions/v2/developers/me", headers, profile)).status ).toBe(200); expect(await listDeveloperHistory(db)).toHaveLength(3); - const current = await getDeveloper(db, profile.id); + const current = await getDeveloper(db, profile.identifier); expect(current?.url).toBeNull(); expect(current?.avatar_url).toBeNull(); expect(current?.contact_email).toBeNull(); @@ -204,13 +210,13 @@ describe("Extensions API v2", () => { const allowed = await put( "/extensions/v2/developers/me", firstHeaders, - { id, type: "user", name: id } + { identifier: id, type: "user", name: id } ); expect(allowed.status).toBe(403); } const denied = await put("/extensions/v2/developers/me", firstHeaders, { - id: "attempt-four", + identifier: "attempt-four", type: "user", name: "Attempt four" }); @@ -228,7 +234,7 @@ describe("Extensions API v2", () => { const otherAccount = await put( "/extensions/v2/developers/me", await authHeaders("independent-rate-limit-account"), - { id: "other-attempt", type: "user", name: "Other attempt" } + { identifier: "other-attempt", type: "user", name: "Other attempt" } ); expect(otherAccount.status).toBe(403); expect(ghRequest).toHaveBeenCalledTimes(4); @@ -260,14 +266,14 @@ describe("Extensions API v2", () => { for (const id of ["remaining-one", "remaining-two"]) { const allowed = await put("/extensions/v2/developers/me", headers, { - id, + identifier: id, type: "user", name: id }); expect(allowed.status).toBe(403); } const denied = await put("/extensions/v2/developers/me", headers, { - id: "no-allowance", + identifier: "no-allowance", type: "user", name: "No allowance" }); @@ -365,7 +371,7 @@ describe("Extensions API v2", () => { if (!raced && sql.includes("INSERT INTO developers")) { raced = true; await insertDeveloper(db, { - id: "raced-developer", + identifier: "raced-developer", type: "user", name: "Concurrent Creator", owner_user_id: "user-2" @@ -417,7 +423,7 @@ describe("Extensions API v2", () => { const res = await put( "/extensions/v2/developers/me", await authHeaders("user-1"), - { id: "acme-org", type: "organization", name: "Acme Org" } + { identifier: "acme-org", type: "organization", name: "Acme Org" } ); expect(res.status).toBe(200); @@ -467,7 +473,7 @@ describe("Extensions API v2", () => { const res = await put( "/extensions/v2/developers/me", await authHeaders("user-1"), - { id: "acme-org", type: "organization", name: "Acme Org" } + { identifier: "acme-org", type: "organization", name: "Acme Org" } ); expect(res.status).toBe(200); @@ -490,7 +496,7 @@ describe("Extensions API v2", () => { const res = await put( "/extensions/v2/developers/me", await authHeaders("user-1"), - { id: "acme-org", type: "organization", name: "Acme Org" } + { identifier: "acme-org", type: "organization", name: "Acme Org" } ); expect(res.status).toBe(200); @@ -512,7 +518,7 @@ describe("Extensions API v2", () => { const res = await put( "/extensions/v2/developers/me", await authHeaders("user-1"), - { id: "acme-org", type: "organization", name: "Acme Org" } + { identifier: "acme-org", type: "organization", name: "Acme Org" } ); expect(res.status).toBe(403); @@ -532,7 +538,7 @@ describe("Extensions API v2", () => { "/extensions/v2/developers/me", await authHeaders("user-1"), { - id: "acme-org", + identifier: "acme-org", type: "organization", name: "Acme Org", URL: "https://acme.example" @@ -564,7 +570,7 @@ describe("Extensions API v2", () => { "/extensions/v2/developers/me", await authHeaders("user-1"), { - id: "acme-org", + identifier: "acme-org", type: "organization", name: "Acme Org", URL: "https://acme.example" @@ -592,7 +598,7 @@ describe("Extensions API v2", () => { "/extensions/v2/developers/me", await authHeaders("user-1"), { - id: "acme-org", + identifier: "acme-org", type: "organization", name: "Acme Org", URL: "http://acme.example/" @@ -617,7 +623,7 @@ describe("Extensions API v2", () => { "/extensions/v2/developers/me", await authHeaders("user-1"), { - id: "acme-org", + identifier: "acme-org", type: "organization", name: "Acme Org", URL: "https://acme.example" @@ -642,7 +648,7 @@ describe("Extensions API v2", () => { "/extensions/v2/developers/me", await authHeaders("user-1"), { - id: "acme-org", + identifier: "acme-org", type: "organization", name: "Acme Org", URL: "https://acme.example/docs" @@ -666,7 +672,7 @@ describe("Extensions API v2", () => { const res = await put( "/extensions/v2/developers/me", await authHeaders("user-1"), - { id: "acme-org", type: "organization", name: "Acme Org" } + { identifier: "acme-org", type: "organization", name: "Acme Org" } ); expect(res.status).toBe(403); @@ -686,7 +692,7 @@ describe("Extensions API v2", () => { const res = await put( "/extensions/v2/developers/me", await authHeaders("user-1"), - { id: "acme-org", type: "user", name: "Acme Org" } + { identifier: "acme-org", type: "user", name: "Acme Org" } ); expect(res.status).toBe(403); @@ -715,7 +721,7 @@ describe("Extensions API v2", () => { const res = await put( "/extensions/v2/developers/me", await authHeaders("user-1"), - { id: "unavailable-dev", type: "user", name: "Unavailable" } + { identifier: "unavailable-dev", type: "user", name: "Unavailable" } ); expect(res.status).toBe(expectedStatus); @@ -739,7 +745,7 @@ describe("Extensions API v2", () => { const res = await put( "/extensions/v2/developers/me", await authHeaders("user-1"), - { id: "unavailable-dev", type: "user", name: "Unavailable" } + { identifier: "unavailable-dev", type: "user", name: "Unavailable" } ); expect(res.status).toBe(503); @@ -761,7 +767,7 @@ describe("Extensions API v2", () => { const res = await put( "/extensions/v2/developers/me", await authHeaders("user-1"), - { id: "invalid-response", type: "user", name: "Invalid" } + { identifier: "invalid-response", type: "user", name: "Invalid" } ); expect(res.status).toBe(503); @@ -779,7 +785,7 @@ describe("Extensions API v2", () => { const res = await put( "/extensions/v2/developers/me", await authHeaders("user-1"), - { id: "unsupported-entity", type: "user", name: "Unsupported" } + { identifier: "unsupported-entity", type: "user", name: "Unsupported" } ); expect(res.status).toBe(422); @@ -795,7 +801,7 @@ describe("Extensions API v2", () => { const res = await put( "/extensions/v2/developers/me", await authHeaders("user-1"), - { id: "acme-org", type: "organization", name: "Acme Org" } + { identifier: "acme-org", type: "organization", name: "Acme Org" } ); expect(res.status).toBe(200); @@ -816,7 +822,7 @@ describe("Extensions API v2", () => { const res = await put( "/extensions/v2/developers/me", await authHeaders("user-1"), - { id: "acme-org", type: "organization", name: "Acme Org" } + { identifier: "acme-org", type: "organization", name: "Acme Org" } ); expect(res.status).toBe(500); @@ -1024,7 +1030,7 @@ describe("Extensions API v2", () => { ); expect( (await listDeveloperHistory(db)).filter( - (row) => row.developer_id === "dev-developer" + (row) => row.name === "Dev Developer" ) ).toHaveLength(1); }); @@ -1204,7 +1210,10 @@ describe("Extensions API v2", () => { .prepare( "INSERT INTO m2m_publisher_clients (client_id, developer_id, can_publish) VALUES (?, ?, 1)" ) - .bind("modules-publisher", "dev-developer") + .bind( + "modules-publisher", + (await getDeveloper(db, "dev-developer"))!.id + ) .run(); const res = await del( @@ -1213,9 +1222,12 @@ describe("Extensions API v2", () => { ); expect(res.status).toBe(200); const body = (await res.json()) as { - result: { id: string; deleted: boolean }; + result: { identifier: string; deleted: boolean }; }; - expect(body.result).toEqual({ id: "dev-developer", deleted: true }); + expect(body.result).toMatchObject({ + id: expect.stringMatching(/^[a-f0-9-]{36}$/), + deleted: true + }); const getRes = await get("/extensions/v2/developers/dev-developer", {}); expect(getRes.status).toBe(404); @@ -1245,7 +1257,9 @@ describe("Extensions API v2", () => { if (!raced && sql.includes("DELETE FROM developers")) { raced = true; await db - .prepare("UPDATE developers SET owner_user_id = ? WHERE id = ?") + .prepare( + "UPDATE developers SET owner_user_id = ? WHERE identifier = ?" + ) .bind("owner-after", "raced-delete") .run(); } @@ -1277,7 +1291,7 @@ describe("Extensions API v2", () => { await insertUser(db, { id: "owner-after" }); await insertDeveloperTransfer(db, { id: "transfer-1", - developer_id: "raced-delete", + developer_identifier: "raced-delete", created_by: "owner-before", token_hash: "hash-1", expires_at: "2099-01-01T00:00:00.000Z" @@ -1288,7 +1302,9 @@ describe("Extensions API v2", () => { if (!raced && sql.includes("DELETE FROM developers")) { raced = true; await db - .prepare("UPDATE developers SET owner_user_id = ? WHERE id = ?") + .prepare( + "UPDATE developers SET owner_user_id = ? WHERE identifier = ?" + ) .bind("owner-after", "raced-delete") .run(); } @@ -1368,7 +1384,7 @@ describe("Extensions API v2", () => { ); await insertDeveloperClaim(db, { id: "claim-1", - developer_id: "dev-developer", + developer_identifier: "dev-developer", claimant_id: "user-2", status: "rejected", review_note: "no", @@ -1376,6 +1392,7 @@ describe("Extensions API v2", () => { reviewed_at: new Date().toISOString() }); + const originalId = (await getDeveloper(db, "dev-developer"))!.id; const res = await del( "/extensions/v2/developers/me", await authHeaders("user-1") @@ -1384,17 +1401,17 @@ describe("Extensions API v2", () => { expect( (await listDeveloperTransfers(db)).filter( - (r) => r.developer_id === "dev-developer" + (r) => r.developer_id === originalId ) ).toHaveLength(0); expect( (await listDeveloperClaims(db)).filter( - (r) => r.developer_id === "dev-developer" + (r) => r.developer_id === originalId ) ).toHaveLength(0); expect( (await listDeveloperHistory(db)).filter( - (r) => r.developer_id === "dev-developer" + (r) => r.developer_id === originalId ).length ).toBeGreaterThan(0); }); @@ -1432,8 +1449,10 @@ describe("Extensions API v2", () => { {} ); expect(stillThere.status).toBe(200); - const body = (await stillThere.json()) as { result: { id: string } }; - expect(body.result.id).toBe("dev-developer"); + const body = (await stillThere.json()) as { + result: { identifier: string }; + }; + expect(body.result.identifier).toBe("dev-developer"); }); it("reports an inactive owner when the account is deactivated during deletion", async () => { @@ -1473,7 +1492,7 @@ describe("Extensions API v2", () => { it("reports an inactive account when deactivated during a URL cooldown reservation", async () => { await insertUser(db, { id: "user-1" }); await insertDeveloper(db, { - id: "dev-developer", + identifier: "dev-developer", type: "organization", name: "Dev", url: "https://acme.example", @@ -1511,7 +1530,7 @@ describe("Extensions API v2", () => { it("re-verifies and refreshes the timestamp when the owner's GitHub org still matches", async () => { await insertDeveloper(db, { - id: "dev-developer", + identifier: "dev-developer", type: "organization", name: "Dev", url: null, @@ -1552,7 +1571,7 @@ describe("Extensions API v2", () => { "preserves verification when the owner's organization evidence is %s", async (_state, github_orgs, github_orgs_expires_at) => { await insertDeveloper(db, { - id: "dev-developer", + identifier: "dev-developer", type: "organization", name: "Dev", url: null, @@ -1587,7 +1606,7 @@ describe("Extensions API v2", () => { it("preserves verification when the owner's GitHub login is whitespace-only", async () => { await insertDeveloper(db, { - id: "dev-developer", + identifier: "dev-developer", type: "organization", name: "Dev", url: null, @@ -1618,7 +1637,7 @@ describe("Extensions API v2", () => { it("flips to unverified when the owner's GitHub org membership no longer matches", async () => { await insertDeveloper(db, { - id: "dev-developer", + identifier: "dev-developer", type: "organization", name: "Dev", url: null, @@ -1654,7 +1673,7 @@ describe("Extensions API v2", () => { it("verifies for the first time on re-check when the caller now has a matching linked GitHub identity", async () => { await insertDeveloper(db, { - id: "dev-developer", + identifier: "dev-developer", type: "organization", name: "Dev", url: null, @@ -1682,7 +1701,7 @@ describe("Extensions API v2", () => { it("doesn't check the Publisher URL without ?check_url=true", async () => { mockGithubEntity("Organization", "https://acme.example"); await insertDeveloper(db, { - id: "dev-developer", + identifier: "dev-developer", type: "organization", name: "Dev", url: "https://acme.example", @@ -1709,7 +1728,7 @@ describe("Extensions API v2", () => { it("checks the Publisher URL when re-verified with ?check_url=true", async () => { mockGithubEntity("Organization", "https://acme.example"); await insertDeveloper(db, { - id: "dev-developer", + identifier: "dev-developer", type: "organization", name: "Dev", url: "https://acme.example", @@ -1739,7 +1758,7 @@ describe("Extensions API v2", () => { "returns an error and retains the cooldown when GitHub responds with %s", async (upstreamStatus, message, expectedStatus, expectedCode) => { await insertDeveloper(db, { - id: "dev-developer", + identifier: "dev-developer", type: "organization", name: "Dev", url: "https://acme.example", @@ -1784,7 +1803,7 @@ describe("Extensions API v2", () => { it("rate-limits repeated ?check_url=true calls from the same caller", async () => { mockGithubEntity("Organization", "https://acme.example"); await insertDeveloper(db, { - id: "dev-developer", + identifier: "dev-developer", type: "organization", name: "Dev", url: "https://acme.example", @@ -1816,7 +1835,7 @@ describe("Extensions API v2", () => { it("doesn't rate-limit reverify calls that don't use ?check_url", async () => { await insertDeveloper(db, { - id: "dev-developer", + identifier: "dev-developer", type: "organization", name: "Dev", url: "https://acme.example", @@ -1843,14 +1862,14 @@ describe("Extensions API v2", () => { it("rate-limits ?check_url=true per caller, not globally", async () => { mockGithubEntity("Organization", "https://acme.example"); await insertDeveloper(db, { - id: "dev-developer", + identifier: "dev-developer", type: "organization", name: "Dev", url: "https://acme.example", owner_user_id: "user-1" }); await insertDeveloper(db, { - id: "other-developer", + identifier: "other-developer", type: "organization", name: "Other", url: "https://acme.example", @@ -1882,7 +1901,7 @@ describe("Extensions API v2", () => { it("only lets one of two concurrent ?check_url=true requests through", async () => { mockGithubEntity("Organization", "https://acme.example"); await insertDeveloper(db, { - id: "dev-developer", + identifier: "dev-developer", type: "organization", name: "Dev", url: "https://acme.example", @@ -1907,7 +1926,7 @@ describe("Extensions API v2", () => { it("does not persist a URL verification computed against a stale URL", async () => { mockGithubEntity("Organization", "https://acme.example"); await insertDeveloper(db, { - id: "dev-developer", + identifier: "dev-developer", type: "organization", name: "Dev", url: "https://acme.example", @@ -1926,7 +1945,7 @@ describe("Extensions API v2", () => { env.DB_EXTENSIONS = wrapD1WithHook(db, async (sql) => { if (sql.includes("developers") && sql.includes("github_verified_at")) { await db - .prepare("UPDATE developers SET url = ? WHERE id = ?") + .prepare("UPDATE developers SET url = ? WHERE identifier = ?") .bind("https://different.example", "dev-developer") .run(); } @@ -1950,7 +1969,7 @@ describe("Extensions API v2", () => { it("clears a previously-verified Publisher URL when identity no longer matches", async () => { mockGithubEntity("Organization", "https://acme.example"); await insertDeveloper(db, { - id: "dev-developer", + identifier: "dev-developer", type: "organization", name: "Dev", url: "https://acme.example", @@ -1982,7 +2001,7 @@ describe("Extensions API v2", () => { it("clears a previously-verified Publisher URL when identity no longer matches, even without ?check_url", async () => { await insertDeveloper(db, { - id: "dev-developer", + identifier: "dev-developer", type: "organization", name: "Dev", url: "https://acme.example", @@ -2022,7 +2041,7 @@ describe("Extensions API v2", () => { // caught separately before trusting the entity's blog field. mockGithubEntity("Organization", "https://acme.example"); await insertDeveloper(db, { - id: "dev-developer", + identifier: "dev-developer", type: "user", name: "Dev", url: "https://acme.example", @@ -2059,7 +2078,7 @@ describe("Extensions API v2", () => { it("preserves an existing Publisher URL verification when the GitHub lookup fails", async () => { mockGithubEntityNotFound(); await insertDeveloper(db, { - id: "dev-developer", + identifier: "dev-developer", type: "organization", name: "Dev", url: "https://acme.example", @@ -2087,7 +2106,7 @@ describe("Extensions API v2", () => { it("treats ?check_url=false the same as omitting it", async () => { mockGithubEntity("Organization", "https://acme.example"); await insertDeveloper(db, { - id: "dev-developer", + identifier: "dev-developer", type: "organization", name: "Dev", url: "https://acme.example", @@ -2121,7 +2140,7 @@ describe("Extensions API v2", () => { it("refuses to overwrite verification if ownership moves away between the lookup and the write", async () => { await insertDeveloper(db, { - id: "dev-developer", + identifier: "dev-developer", type: "organization", name: "Dev", url: null, @@ -2157,7 +2176,7 @@ describe("Extensions API v2", () => { it("refuses to overwrite verification if the profile type changes during the check", async () => { await insertDeveloper(db, { - id: "dev-developer", + identifier: "dev-developer", type: "organization", name: "Dev", url: null, @@ -2178,7 +2197,7 @@ describe("Extensions API v2", () => { if (!changed && sql.includes("github_verified_at")) { changed = true; await db - .prepare("UPDATE developers SET type = ? WHERE id = ?") + .prepare("UPDATE developers SET type = ? WHERE identifier = ?") .bind("user", "dev-developer") .run(); } @@ -2200,7 +2219,7 @@ describe("Extensions API v2", () => { it("refuses to overwrite verification if GitHub identity sync wins the race", async () => { await insertDeveloper(db, { - id: "dev-developer", + identifier: "dev-developer", type: "organization", name: "Dev", url: null, @@ -2256,7 +2275,7 @@ describe("Extensions API v2", () => { describe("GET /developers/{id}", () => { it("returns a developer's public profile without contact_email, unauthenticated", async () => { await insertDeveloper(db, { - id: "public-dev", + identifier: "public-dev", type: "organization", name: "Public Dev", url: "https://example.com", @@ -2270,7 +2289,8 @@ describe("Extensions API v2", () => { expect(res.status).toBe(200); const body = (await res.json()) as { result: Record }; expect(body.result).toEqual({ - id: "public-dev", + id: expect.stringMatching(UUID_V7), + identifier: "public-dev", type: "organization", name: "Public Dev", URL: "https://example.com", @@ -2292,13 +2312,13 @@ describe("Extensions API v2", () => { const res = await get("/extensions/v2/developers/legacy-public", {}); expect(res.status).toBe(200); expect(await res.json()).toMatchObject({ - result: { id: "legacy-public", unclaimed: true } + result: { identifier: "legacy-public", unclaimed: true } }); }); it("falls back to the public view for a deactivated owner", async () => { await insertDeveloper(db, { - id: "owned-dev", + identifier: "owned-dev", type: "user", name: "Owned Dev", contact_email: "private@example.com", @@ -2317,7 +2337,10 @@ describe("Extensions API v2", () => { expect(res.status).toBe(200); const body = (await res.json()) as { result: Record }; expect(body.result.contact_email).toBeUndefined(); - expect(body.result).toMatchObject({ id: "owned-dev", unclaimed: false }); + expect(body.result).toMatchObject({ + identifier: "owned-dev", + unclaimed: false + }); expect(res.headers.get("Vary")).toBe("Authorization"); }); }); diff --git a/test/services/extensions/v2/directory-identity.test.ts b/test/services/extensions/v2/directory-identity.test.ts new file mode 100644 index 00000000..3f8bb8ad --- /dev/null +++ b/test/services/extensions/v2/directory-identity.test.ts @@ -0,0 +1,95 @@ +import { UUID_V7 } from "./db-fixtures"; +import { describe, expect, it, vi } from "vitest"; +import { + authHeaders, + db, + get, + post, + put, + sampleCreate, + sampleDeveloper, + setupExtensionsV2Tests +} from "./harness"; + +vi.mock("@octokit/request", async () => + (await import("../../../mocks/octokit")).octokitRequestMock() +); +setupExtensionsV2Tests(); + +describe("directory identity", () => { + it("generates UUIDv7 keys and keeps public identifiers out of foreign keys", async () => { + const headers = await authHeaders("owner"); + const profile = await put( + "/extensions/v2/developers/me", + headers, + sampleDeveloper() + ); + expect(profile.status).toBe(200); + const developer = ( + (await profile.json()) as { result: { id: string; identifier: string } } + ).result; + expect(developer.id).toMatch(UUID_V7); + expect(developer.identifier).toBe("dev-developer"); + const created = await post( + "/extensions/v2/extensions", + headers, + sampleCreate() + ); + expect(created.status).toBe(201); + const extension = ( + (await created.json()) as { + result: { id: string; identifier: string; revision_id: string }; + } + ).result; + expect(extension.id).toMatch(UUID_V7); + expect(extension.identifier).toBe("new-ext"); + expect( + await db + .prepare("SELECT developer_id FROM extensions WHERE id=?") + .bind(extension.id) + .first() + ).toEqual({ developer_id: developer.id }); + expect( + await db + .prepare( + "SELECT extension_id,developer_id FROM extension_revisions WHERE id=?" + ) + .bind(extension.revision_id) + .first() + ).toEqual({ extension_id: extension.id, developer_id: developer.id }); + expect( + (await get("/extensions/v2/extensions/new-ext", headers)).status + ).toBe(200); + expect( + (await get(`/extensions/v2/extensions/${extension.id}`, headers)).status + ).toBe(404); + expect( + (await get("/extensions/v2/developers/dev-developer", headers)).status + ).toBe(200); + }); + + it("preserves both identities when display names change", async () => { + const headers = await authHeaders("owner"); + const response = await put( + "/extensions/v2/developers/me", + headers, + sampleDeveloper() + ); + const before = ( + (await response.json()) as { result: { id: string; identifier: string } } + ).result; + const updated = await put( + "/extensions/v2/developers/me", + headers, + sampleDeveloper({ name: "New Display Name" }) + ); + expect(updated.status).toBe(200); + expect( + ((await updated.json()) as { result: unknown }).result + ).toMatchObject({ + id: before.id, + identifier: before.identifier, + name: "New Display Name" + }); + }); +}); diff --git a/test/services/extensions/v2/extension-writes.test.ts b/test/services/extensions/v2/extension-writes.test.ts index 73103c82..5fccf96c 100644 --- a/test/services/extensions/v2/extension-writes.test.ts +++ b/test/services/extensions/v2/extension-writes.test.ts @@ -20,6 +20,7 @@ import { countRevisions, insertUser, getExtension, + getDeveloper, getRevision, insertDeveloper, insertExtension, @@ -38,14 +39,14 @@ setupExtensionsV2Tests(); // lowercaseId() rejects them at validation. async function seedMixedCaseExtension() { await insertDeveloper(db, { - id: "owner-developer", + identifier: "owner-developer", type: "user", name: "Owner", owner_user_id: "owner-1" }); await insertExtension(db, { - id: "Existing-Ext", - developer_id: "owner-developer", + identifier: "Existing-Ext", + developer_identifier: "owner-developer", name: "Existing" }); } @@ -156,15 +157,23 @@ describe("Extensions API v2 writes", () => { expect(res.status).toBe(201); const data = (await res.json()) as { - result: { id: string; revision_id: string; status: string }; + result: { + id: string; + identifier: string; + revision_id: string; + status: string; + }; }; - expect(data.result).toMatchObject({ id: "new-ext", status: "pending" }); + expect(data.result).toMatchObject({ + identifier: "new-ext", + status: "pending" + }); // Holds its id immediately, but stays unpublished until a moderator // approves. const stored = await getExtension(db, "new-ext"); expect(stored).toMatchObject({ - developer_id: "new-developer", + developer_id: (await getDeveloper(db, "new-developer"))!.id, published_at: null, published_revision_id: null, name: null @@ -172,7 +181,7 @@ describe("Extensions API v2 writes", () => { const revision = await getRevision(db, data.result.revision_id); expect(revision).toMatchObject({ - extension_id: "new-ext", + extension_id: (await getExtension(db, "new-ext"))!.id, submitted_by: "user-1", status: "pending" }); @@ -315,7 +324,7 @@ describe("Extensions API v2 writes", () => { // The edit body carries the same guard. await seedOwnedExtension(); - const { id: _id, ...content } = body; + const { identifier: _identifier, ...content } = body; const edited = await put( "/extensions/v2/extensions/existing-ext", await authHeaders("owner-1"), @@ -400,7 +409,7 @@ describe("Extensions API v2 writes", () => { expect(res.status).toBe(202); const [revision] = await listRevisions(db); expect(revision).toMatchObject({ - extension_id: "existing-ext", + extension_id: (await getExtension(db, "existing-ext"))!.id, status: "pending" }); expect(JSON.parse(revision.content).name).toBe("Renamed"); @@ -516,7 +525,9 @@ describe("Extensions API v2 writes", () => { expect(res.status).toBe(202); // The revision must hang off the stored spelling, not the requested one. const [revision] = await listRevisions(db); - expect(revision.extension_id).toBe("Existing-Ext"); + expect(revision.extension_id).toBe( + (await getExtension(db, "Existing-Ext"))!.id + ); }); it("lists revisions addressed in lower case", async () => { @@ -730,13 +741,14 @@ describe("Extensions API v2 writes", () => { const data = (await res.json()) as { result: Array<{ id: string; + identifier: string; published: { name: string } | null; - pending_revision: { id: string } | null; + pending_revision: { id: string; identifier: string } | null; last_review: unknown; }>; }; - expect(data.result.map((item) => item.id)).toEqual([ + expect(data.result.map((item) => item.identifier)).toEqual([ "draft-ext", "existing-ext" ]); @@ -761,7 +773,7 @@ describe("Extensions API v2 writes", () => { const data = (await res.json()) as { result: Array<{ published: { name: string } | null; - pending_revision: { id: string } | null; + pending_revision: { id: string; identifier: string } | null; }>; }; expect(data.result[0].published).toMatchObject({ name: "Existing" }); @@ -833,8 +845,12 @@ describe("Extensions API v2 writes", () => { "/extensions/v2/extensions?scope=mine", await authHeaders("owner-1") ); - const data = (await res.json()) as { result: Array<{ id: string }> }; - expect(data.result.map((item) => item.id)).toEqual(["existing-ext"]); + const data = (await res.json()) as { + result: Array<{ id: string; identifier: string }>; + }; + expect(data.result.map((item) => item.identifier)).toEqual([ + "existing-ext" + ]); }); // extensions.type is NULL until a first approval, so filtering the column @@ -876,8 +892,10 @@ describe("Extensions API v2 writes", () => { headers ); expect(res.status).toBe(200); - const data = (await res.json()) as { result: Array<{ id: string }> }; - expect(data.result.map((item) => item.id)).toEqual([ + const data = (await res.json()) as { + result: Array<{ id: string; identifier: string }>; + }; + expect(data.result.map((item) => item.identifier)).toEqual([ "draft-ext", "live-ext", "rejected-ext" @@ -920,10 +938,10 @@ describe("Extensions API v2 writes", () => { headers ); const firstBody = (await first.json()) as { - result: Array<{ id: string }>; + result: Array<{ id: string; identifier: string }>; pagination: { has_more: boolean; next_cursor: string }; }; - expect(firstBody.result.map((item) => item.id)).toEqual([ + expect(firstBody.result.map((item) => item.identifier)).toEqual([ "page-a", "page-b" ]); @@ -934,10 +952,12 @@ describe("Extensions API v2 writes", () => { headers ); const secondBody = (await second.json()) as { - result: Array<{ id: string }>; + result: Array<{ id: string; identifier: string }>; pagination: { has_more: boolean; next_cursor: null }; }; - expect(secondBody.result.map((item) => item.id)).toEqual(["page-c"]); + expect(secondBody.result.map((item) => item.identifier)).toEqual([ + "page-c" + ]); expect(secondBody.pagination).toEqual({ has_more: false, next_cursor: null @@ -973,14 +993,14 @@ describe("Extensions API v2 writes", () => { // advertised schema catches the disagreement. it("serves an adopted extension with no releases against its own schema", async () => { await insertDeveloper(db, { - id: "owner-developer", + identifier: "owner-developer", type: "user", name: "Owner", owner_user_id: "owner-1" }); await insertExtension(db, { - id: "adopted-ext", - developer_id: "owner-developer", + identifier: "adopted-ext", + developer_identifier: "owner-developer", name: "Adopted", releases: "[]" }); @@ -1006,9 +1026,14 @@ describe("Extensions API v2 writes", () => { ); expect(res.status).toBe(200); const data = (await res.json()) as { - result: { id: string; readme: string; pending_revision?: unknown }; + result: { + id: string; + identifier: string; + readme: string; + pending_revision?: unknown; + }; }; - expect(data.result.id).toBe("existing-ext"); + expect(data.result.identifier).toBe("existing-ext"); expect(data.result.readme).toBeDefined(); expect("pending_revision" in data.result).toBe(false); }); @@ -1043,10 +1068,14 @@ describe("Extensions API v2 writes", () => { ); expect(res.status).toBe(200); const data = (await res.json()) as { - result: Array<{ extension_id: string }>; + result: Array<{ + extension_id: string; + identifier: string; + extension_identifier: string; + }>; }; expect(data.result).toHaveLength(1); - expect(data.result[0].extension_id).toBe("mine"); + expect(data.result[0].extension_identifier).toBe("mine"); }); }); @@ -1062,11 +1091,15 @@ describe("Extensions API v2 writes", () => { ); expect(res.status).toBe(200); const data = (await res.json()) as { - result: Array<{ extension_id: string; status: string }>; + result: Array<{ + extension_id: string; + identifier: string; + status: string; + }>; }; expect(data.result).toHaveLength(1); expect(data.result[0]).toMatchObject({ - extension_id: "new-ext", + extension_id: (await getExtension(db, "new-ext"))!.id, status: "pending" }); }); diff --git a/test/services/extensions/v2/harness.ts b/test/services/extensions/v2/harness.ts index a66e0012..c7e77ff6 100644 --- a/test/services/extensions/v2/harness.ts +++ b/test/services/extensions/v2/harness.ts @@ -158,14 +158,14 @@ export function sampleCreate(overrides?: { name?: string; }) { return { - id: overrides?.extensionId ?? "new-ext", + identifier: overrides?.extensionId ?? "new-ext", ...sampleContent(overrides) }; } export function sampleDeveloper(overrides?: { id?: string; name?: string }) { return { - id: overrides?.id ?? "dev-developer", + identifier: overrides?.id ?? "dev-developer", type: "user", name: overrides?.name ?? "Dev Developer", URL: "https://example.com" @@ -179,7 +179,7 @@ export async function seedDeveloper( ownerUserId: string ): Promise { await insertDeveloper(db, { - id, + identifier: id, type: "user", name: "Developer", url: null, @@ -192,7 +192,7 @@ export async function seedUnownedDeveloper( name = "Legacy Developer" ): Promise { await insertDeveloper(db, { - id, + identifier: id, type: "user", name, url: null, @@ -202,15 +202,15 @@ export async function seedUnownedDeveloper( export async function seedOwnedExtension(): Promise { await insertDeveloper(db, { - id: "owner-developer", + identifier: "owner-developer", type: "user", name: "Owner", url: null, owner_user_id: "owner-1" }); await insertExtension(db, { - id: "existing-ext", - developer_id: "owner-developer", + identifier: "existing-ext", + developer_identifier: "owner-developer", name: "Existing", source: '{"type":"github","repo":"example/existing"}' }); diff --git a/test/services/extensions/v2/manifest.test.ts b/test/services/extensions/v2/manifest.test.ts index a4850efe..9eb547cb 100644 --- a/test/services/extensions/v2/manifest.test.ts +++ b/test/services/extensions/v2/manifest.test.ts @@ -54,7 +54,10 @@ describe("manifest v1 publishing", () => { const { result: extension } = (await detail.json()) as { result: Record; }; - expect(extension).toMatchObject({ ...content, developer: { id: "owner" } }); + expect(extension).toMatchObject({ + ...content, + developer: { identifier: "owner" } + }); expect(extension).not.toHaveProperty("website"); const list = await get("/extensions/v2/extensions?type=module", {}); const { result: items } = (await list.json()) as { @@ -126,18 +129,20 @@ describe("manifest v1 publishing", () => { expect( ( await post( - `/extensions/v2/extensions/${content.id}/revisions/${result.revision_id}/approve`, + `/extensions/v2/extensions/${content.identifier}/revisions/${result.revision_id}/approve`, await authHeaders("moderator"), {} ) ).status ).toBe(200); const published = (await ( - await get(`/extensions/v2/extensions/${content.id}`, {}) + await get(`/extensions/v2/extensions/${content.identifier}`, {}) ).json()) as { result: typeof content }; expect(published.result.requires).toEqual(requires); expect(published.result.releases[0].requires).toEqual(requires); - expect((await get(`/extensions/v1/${content.id}`, {})).status).toBe(200); + expect((await get(`/extensions/v1/${content.identifier}`, {})).status).toBe( + 200 + ); }); it("keeps the historical minimum-version field bounded in revision schemas", () => { @@ -188,7 +193,10 @@ describe("manifest v1 publishing", () => { it("allows existing long identities to be edited without relaxing new manifest IDs", async () => { await seedDeveloper("owner", "user-1"); const id = "e".repeat(120); - await insertExtension(db, { id, developer_id: "owner" }); + await insertExtension(db, { + identifier: id, + developer_identifier: "owner" + }); expect( ( await put( diff --git a/test/services/extensions/v2/migrations.test.ts b/test/services/extensions/v2/migrations.test.ts index 9b8f5a47..8dbfb235 100644 --- a/test/services/extensions/v2/migrations.test.ts +++ b/test/services/extensions/v2/migrations.test.ts @@ -1374,3 +1374,68 @@ it.each(["", "v"])( } } ); + +describe("directory UUID identities", () => { + it("preserves populated relationships and resource accounting in a D1 transaction", () => { + const db = new DatabaseSync(":memory:"); + try { + db.exec("PRAGMA foreign_keys=ON"); + for (const name of migrationNames.filter((name) => name < "0032")) + db.exec(migration(name)); + db.exec(` + INSERT INTO users(id,created_at,updated_at) VALUES ('owner',CURRENT_TIMESTAMP,CURRENT_TIMESTAMP); + INSERT INTO developers(id,type,name,owner_user_id,created_at) VALUES ('studio','user','Studio','owner','2026-11-01 00:00:00'); + INSERT INTO extensions(id,developer_id,created_by) VALUES ('example','studio','owner'); + INSERT INTO extension_revisions(id,extension_id,developer_id,submitted_by,content) VALUES ('revision','example','studio','owner','{}'); + INSERT INTO developer_claims(id,developer_id,claimant_id) VALUES ('claim','studio','owner'); + INSERT INTO m2m_publisher_clients(client_id,developer_id,can_publish) VALUES ('publisher','studio',1); + `); + const before = db + .prepare( + "SELECT bytes,extensions,revisions FROM extension_resource_usage WHERE scope='developer' AND subject='studio'" + ) + .get(); + db.exec("BEGIN"); + db.exec(migration("0032_directory_identity.sql")); + db.exec("COMMIT"); + const developer = db + .prepare("SELECT id,identifier FROM developers") + .get() as { id: string; identifier: string }; + const extension = db + .prepare("SELECT id,identifier,developer_id FROM extensions") + .get() as { id: string; identifier: string; developer_id: string }; + const pattern = + /^[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/; + expect(developer).toEqual({ + id: expect.stringMatching(pattern), + identifier: "studio" + }); + expect(extension).toEqual({ + id: expect.stringMatching(pattern), + identifier: "example", + developer_id: developer.id + }); + expect( + db + .prepare("SELECT extension_id,developer_id FROM extension_revisions") + .get() + ).toEqual({ extension_id: extension.id, developer_id: developer.id }); + expect( + db.prepare("SELECT developer_id FROM developer_claims").get() + ).toEqual({ developer_id: developer.id }); + expect( + db.prepare("SELECT developer_id FROM m2m_publisher_clients").get() + ).toEqual({ developer_id: developer.id }); + expect( + db + .prepare( + "SELECT bytes,extensions,revisions FROM extension_resource_usage WHERE scope='developer' AND subject=?" + ) + .get(developer.id) + ).toEqual(before); + expect(db.prepare("PRAGMA foreign_key_check").all()).toEqual([]); + } finally { + db.close(); + } + }); +}); diff --git a/test/services/extensions/v2/moderation-correct.test.ts b/test/services/extensions/v2/moderation-correct.test.ts index 79f74c81..420ee2ac 100644 --- a/test/services/extensions/v2/moderation-correct.test.ts +++ b/test/services/extensions/v2/moderation-correct.test.ts @@ -1,3 +1,4 @@ +import { UUID_V7 } from "./db-fixtures"; import { describe, it, expect, vi } from "vitest"; import { env } from "cloudflare:workers"; import { wrapD1WithHook } from "./db-interceptor"; @@ -44,8 +45,8 @@ async function seedPublished(): Promise { await insertUser(db, { id: "mod-1", is_moderator: 1 }); await seedDeveloper("new-developer", "user-1"); await insertExtension(db, { - id: "live-ext", - developer_id: "new-developer", + identifier: "live-ext", + developer_identifier: "new-developer", readme: "old readme" }); } @@ -64,7 +65,7 @@ describe("POST /extensions/{id}/moderator-correct (api#251)", () => { ); expect(response.status).toBe(200); await expect(response.json()).resolves.toMatchObject({ - result: { id: "live-ext" } + result: { id: expect.stringMatching(UUID_V7), identifier: "live-ext" } }); expect(await countRevisions(db)).toBe(1); }); @@ -169,8 +170,8 @@ describe("POST /extensions/{id}/moderator-correct (api#251)", () => { await insertUser(db, { id: "mod-1", is_moderator: 1 }); await seedDeveloper("new-developer", "user-1"); await insertUnpublishedExtension(db, { - id: "draft-ext", - developer_id: "new-developer" + identifier: "draft-ext", + developer_identifier: "new-developer" }); const res = await post( "/extensions/v2/extensions/draft-ext/moderator-correct", @@ -188,7 +189,7 @@ describe("POST /extensions/{id}/moderator-correct (api#251)", () => { await seedPublished(); await db .prepare( - "UPDATE extensions SET delisted_at = ?, delist_reason = ? WHERE id = ?" + "UPDATE extensions SET delisted_at = ?, delist_reason = ? WHERE identifier= ?" ) .bind(new Date().toISOString(), "cause", "live-ext") .run(); @@ -212,7 +213,11 @@ describe("POST /extensions/{id}/moderator-correct (api#251)", () => { const res = await post(PATH, await authHeaders("mod-1"), correctBody()); expect(res.status).toBe(409); await expect(res.json()).resolves.toMatchObject({ - error: { code: "CONFLICT" } + error: { + code: "CONFLICT", + message: + "An edit to this extension is already awaiting review; approve or reject it first" + } }); // Neither the live content nor the pending edit moves. expect((await getExtension(db, "live-ext"))?.readme).toBe("old readme"); @@ -264,9 +269,18 @@ describe("POST /extensions/{id}/moderator-correct (api#251)", () => { ); expect(res.status).toBe(200); const body = (await res.json()) as { - result: { id: string; revision_id: string; status: string }; + result: { + id: string; + identifier: string; + revision_id: string; + status: string; + }; }; - expect(body.result).toMatchObject({ id: "live-ext", status: "approved" }); + expect(body.result).toMatchObject({ + id: expect.stringMatching(UUID_V7), + identifier: "live-ext", + status: "approved" + }); expect(typeof body.result.revision_id).toBe("string"); // No notification envelope on this route (api#251: history only). expect("notified" in body.result).toBe(false); @@ -279,7 +293,7 @@ describe("POST /extensions/{id}/moderator-correct (api#251)", () => { const revision = await getRevision(db, body.result.revision_id); expect(revision).toMatchObject({ - extension_id: "live-ext", + extension_id: expect.stringMatching(UUID_V7), status: "approved", submitted_by: "mod-1", reviewer_id: "mod-1", @@ -292,7 +306,8 @@ describe("POST /extensions/{id}/moderator-correct (api#251)", () => { // Ownership is untouched: developer, owner, and epoch survive. expect(await getDeveloper(db, "new-developer")).toMatchObject({ - id: "new-developer", + id: expect.stringMatching(UUID_V7), + identifier: "new-developer", owner_user_id: "user-1", ownership_epoch: 1 }); @@ -305,8 +320,8 @@ describe("POST /extensions/{id}/moderator-correct (api#251)", () => { await insertUser(db, { id: "mod-1", is_moderator: 1 }); await seedDeveloper("new-developer", "user-1"); await insertExtension(db, { - id: "LIVE-ext", - developer_id: "new-developer" + identifier: "LIVE-ext", + developer_identifier: "new-developer" }); const res = await post( "/extensions/v2/extensions/live-EXT/moderator-correct", @@ -315,7 +330,11 @@ describe("POST /extensions/{id}/moderator-correct (api#251)", () => { ); expect(res.status).toBe(200); await expect(res.json()).resolves.toMatchObject({ - result: { id: "LIVE-ext", status: "approved" } + result: { + id: expect.stringMatching(UUID_V7), + identifier: "LIVE-ext", + status: "approved" + } }); }); @@ -329,12 +348,12 @@ describe("POST /extensions/{id}/moderator-correct (api#251)", () => { ); expect(created.status).toBe(201); const { result: pending } = (await created.json()) as { - result: { id: string; revision_id: string }; + result: { id: string; identifier: string; revision_id: string }; }; expect( ( await post( - `/extensions/v2/extensions/${pending.id}/revisions/${pending.revision_id}/approve?notify=false`, + `/extensions/v2/extensions/${pending.identifier}/revisions/${pending.revision_id}/approve?notify=false`, await authHeaders("mod-1"), {} ) @@ -353,7 +372,7 @@ describe("POST /extensions/{id}/moderator-correct (api#251)", () => { const revisions = await listRevisions(db); expect( revisions - .filter((r) => r.extension_id === "history-ext") + .filter((r) => r.extension_id === pending.id) .map((r) => r.status) .sort() ).toEqual(["approved", "approved"]); diff --git a/test/services/extensions/v2/moderation-notify.test.ts b/test/services/extensions/v2/moderation-notify.test.ts index aa99770b..52253ca3 100644 --- a/test/services/extensions/v2/moderation-notify.test.ts +++ b/test/services/extensions/v2/moderation-notify.test.ts @@ -1,3 +1,4 @@ +import { UUID_V7 } from "./db-fixtures"; import { describe, it, expect, vi, afterEach } from "vitest"; import { env } from "cloudflare:workers"; import { @@ -58,21 +59,24 @@ function stubSmtpApi(payload: unknown = { success: true, message: "sent" }) { async function seedLiveExtension(): Promise { await insertDeveloper(db, { - id: "new-developer", + identifier: "new-developer", type: "user", name: "New Developer", url: null, owner_user_id: "user-1", contact_email: "author@example.com" }); - await insertExtension(db, { id: "live-ext", developer_id: "new-developer" }); + await insertExtension(db, { + identifier: "live-ext", + developer_identifier: "new-developer" + }); } async function seedPendingClaim(): Promise { await insertUser(db, { id: "mod-1", is_moderator: 1 }); await insertUser(db, { id: "claimant-1", email: "claimant@example.com" }); await insertDeveloper(db, { - id: "legacy-dev", + identifier: "legacy-dev", type: "user", name: "Legacy", url: null, @@ -80,7 +84,7 @@ async function seedPendingClaim(): Promise { }); await insertDeveloperClaim(db, { id: "claim-1", - developer_id: "legacy-dev", + developer_identifier: "legacy-dev", claimant_id: "claimant-1" }); } @@ -99,7 +103,12 @@ describe("moderation notification emails", () => { ); expect(res.status).toBe(200); await expect(res.json()).resolves.toEqual({ - result: { id: "live-ext", status: "delisted", notified: true } + result: { + id: expect.stringMatching(UUID_V7), + identifier: "live-ext", + status: "delisted", + notified: true + } }); expect(calls).toHaveLength(1); @@ -132,7 +141,12 @@ describe("moderation notification emails", () => { ); expect(res.status).toBe(200); await expect(res.json()).resolves.toEqual({ - result: { id: "live-ext", status: "relisted", notified: true } + result: { + id: expect.stringMatching(UUID_V7), + identifier: "live-ext", + status: "relisted", + notified: true + } }); expect(calls).toHaveLength(1); @@ -207,7 +221,12 @@ describe("moderation notification emails", () => { ); expect(res.status).toBe(200); await expect(res.json()).resolves.toEqual({ - result: { id: "live-ext", status: "delisted", notified: false } + result: { + id: expect.stringMatching(UUID_V7), + identifier: "live-ext", + status: "delisted", + notified: false + } }); expect(calls).toHaveLength(0); }); @@ -216,15 +235,15 @@ describe("moderation notification emails", () => { await insertUser(db, { id: "mod-1", is_moderator: 1 }); await insertUser(db, { id: "user-1", email: "owner@example.com" }); await insertDeveloper(db, { - id: "new-developer", + identifier: "new-developer", type: "user", name: "New Developer", url: null, owner_user_id: "user-1" }); await insertExtension(db, { - id: "live-ext", - developer_id: "new-developer" + identifier: "live-ext", + developer_identifier: "new-developer" }); setEmailEnv(); const calls = stubSmtpApi(); @@ -262,7 +281,12 @@ describe("moderation notification emails", () => { // it. The provider failure is visible only in logs; the delist itself // must still succeed. await expect(res.json()).resolves.toEqual({ - result: { id: "live-ext", status: "delisted", notified: true } + result: { + id: expect.stringMatching(UUID_V7), + identifier: "live-ext", + status: "delisted", + notified: true + } }); expect(calls).toHaveLength(1); expect((await getExtension(db, "live-ext"))?.delist_reason).toBe( @@ -277,7 +301,7 @@ describe("moderation notification emails", () => { await insertUser(db, { id: "mod-1", is_moderator: 1 }); await insertUser(db, { id: "user-1", email: "original@example.com" }); await insertDeveloper(db, { - id: "new-developer", + identifier: "new-developer", type: "user", name: "New Developer", url: null, @@ -290,7 +314,7 @@ describe("moderation notification emails", () => { ); expect(created.status).toBe(201); const { result } = (await created.json()) as { - result: { id: string; revision_id: string }; + result: { id: string; identifier: string; revision_id: string }; }; await bumpDeveloperOwnership(db, "new-developer", "user-2"); await insertUser(db, { id: "user-2", email: "current@example.com" }); @@ -298,7 +322,7 @@ describe("moderation notification emails", () => { const calls = stubSmtpApi(); const res = await post( - `/extensions/v2/extensions/${result.id}/revisions/${result.revision_id}/reject`, + `/extensions/v2/extensions/${result.identifier}/revisions/${result.revision_id}/reject`, await authHeaders("mod-1"), { review_note: "Needs a valid license URL" } ); @@ -350,15 +374,15 @@ describe("moderation notification emails", () => { it("reports notified:false when no address exists anywhere", async () => { await insertUser(db, { id: "mod-1", is_moderator: 1 }); await insertDeveloper(db, { - id: "new-developer", + identifier: "new-developer", type: "user", name: "New Developer", url: null, owner_user_id: null }); await insertExtension(db, { - id: "live-ext", - developer_id: "new-developer" + identifier: "live-ext", + developer_identifier: "new-developer" }); setEmailEnv(); const calls = stubSmtpApi(); @@ -403,7 +427,7 @@ describe("moderation notification emails", () => { await insertUser(db, { id: "mod-1", is_moderator: 1 }); await insertUser(db, { id: "user-1", email: "owner@example.com" }); await insertDeveloper(db, { - id: "new-developer", + identifier: "new-developer", type: "user", name: "New Developer", url: null, @@ -411,8 +435,8 @@ describe("moderation notification emails", () => { contact_email: "The Owner " }); await insertExtension(db, { - id: "live-ext", - developer_id: "new-developer" + identifier: "live-ext", + developer_identifier: "new-developer" }); setEmailEnv(); const calls = stubSmtpApi(); @@ -434,7 +458,7 @@ describe("moderation notification emails", () => { await insertUser(db, { id: "mod-1", is_moderator: 1 }); await insertUser(db, { id: "user-1", email: "owner@example.com" }); await insertDeveloper(db, { - id: "new-developer", + identifier: "new-developer", type: "user", name: "New Developer", url: null, @@ -447,13 +471,13 @@ describe("moderation notification emails", () => { ); expect(created.status).toBe(201); const { result } = (await created.json()) as { - result: { id: string; revision_id: string }; + result: { id: string; identifier: string; revision_id: string }; }; setEmailEnv(); const calls = stubSmtpApi(); const res = await post( - `/extensions/v2/extensions/${result.id}/revisions/${result.revision_id}/approve`, + `/extensions/v2/extensions/${result.identifier}/revisions/${result.revision_id}/approve`, await authHeaders("mod-1"), { review_note: " " } ); diff --git a/test/services/extensions/v2/moderation-revalidate.test.ts b/test/services/extensions/v2/moderation-revalidate.test.ts index fdbad7ee..f2b3fb7f 100644 --- a/test/services/extensions/v2/moderation-revalidate.test.ts +++ b/test/services/extensions/v2/moderation-revalidate.test.ts @@ -1,3 +1,4 @@ +import { UUID_V7 } from "./db-fixtures"; import { describe, it, expect, vi, afterEach } from "vitest"; import { env } from "cloudflare:workers"; import { listCacheKey } from "../../../../src/services/extensions/v1/list-cache"; @@ -61,15 +62,15 @@ async function seedModAndExtension(): Promise { await insertUser(db, { id: "mod-1", is_moderator: 1 }); await insertUser(db, { id: "user-1", email: "owner@example.com" }); await insertDeveloper(db, { - id: "new-developer", + identifier: "new-developer", type: "user", name: "New Developer", url: null, owner_user_id: "user-1" }); await insertExtension(db, { - id: "live-ext", - developer_id: "new-developer" + identifier: "live-ext", + developer_identifier: "new-developer" }); } @@ -181,12 +182,12 @@ describe("CDN cache revalidation on extension mutations", () => { ); expect(created.status).toBe(201); const { result } = (await created.json()) as { - result: { id: string; revision_id: string }; + result: { id: string; identifier: string; revision_id: string }; }; const fetcher = stubFrontend(); const res = await post( - `/extensions/v2/extensions/${result.id}/revisions/${result.revision_id}/approve?notify=false`, + `/extensions/v2/extensions/${result.identifier}/revisions/${result.revision_id}/approve?notify=false`, await authHeaders("mod-1"), {} ); @@ -216,12 +217,12 @@ describe("CDN cache revalidation on extension mutations", () => { ); expect(created.status).toBe(201); const { result } = (await created.json()) as { - result: { id: string; revision_id: string }; + result: { id: string; identifier: string; revision_id: string }; }; const fetcher = stubFrontend(); const res = await post( - `/extensions/v2/extensions/${result.id}/revisions/${result.revision_id}/reject?notify=false`, + `/extensions/v2/extensions/${result.identifier}/revisions/${result.revision_id}/reject?notify=false`, await authHeaders("mod-1"), { review_note: "Needs work" } ); @@ -233,7 +234,7 @@ describe("CDN cache revalidation on extension mutations", () => { await insertUser(db, { id: "mod-1", is_moderator: 1 }); await insertUser(db, { id: "claimant-1", email: "claimant@example.com" }); await insertDeveloper(db, { - id: "legacy-dev", + identifier: "legacy-dev", type: "user", name: "Legacy", url: null, @@ -241,7 +242,7 @@ describe("CDN cache revalidation on extension mutations", () => { }); await insertDeveloperClaim(db, { id: "claim-1", - developer_id: "legacy-dev", + developer_identifier: "legacy-dev", claimant_id: "claimant-1" }); const fetcher = stubFrontend(); @@ -257,15 +258,15 @@ describe("CDN cache revalidation on extension mutations", () => { it("purges after an owner withdraws an unpublished extension", async () => { await insertUser(db, { id: "user-1", email: "owner@example.com" }); await insertDeveloper(db, { - id: "new-developer", + identifier: "new-developer", type: "user", name: "New Developer", url: null, owner_user_id: "user-1" }); await insertUnpublishedExtension(db, { - id: "draft-ext", - developer_id: "new-developer" + identifier: "draft-ext", + developer_identifier: "new-developer" }); const fetcher = stubFrontend(); @@ -294,7 +295,7 @@ describe("CDN cache revalidation on extension mutations", () => { // still public content, so the delete path must purge too. await insertUser(db, { id: "user-1", email: "owner@example.com" }); await insertDeveloper(db, { - id: "doomed-dev", + identifier: "doomed-dev", type: "user", name: "Doomed", url: null, @@ -329,7 +330,11 @@ describe("CDN cache revalidation on extension mutations", () => { const res = await delist("mod-1"); expect(res.status).toBe(200); await expect(res.json()).resolves.toMatchObject({ - result: { id: "live-ext", status: "delisted" } + result: { + id: expect.stringMatching(UUID_V7), + identifier: "live-ext", + status: "delisted" + } }); }); diff --git a/test/services/extensions/v2/moderation-revocation.test.ts b/test/services/extensions/v2/moderation-revocation.test.ts index 4c45ff86..4d11500b 100644 --- a/test/services/extensions/v2/moderation-revocation.test.ts +++ b/test/services/extensions/v2/moderation-revocation.test.ts @@ -52,30 +52,30 @@ describe("commit-time moderator authority", () => { await insertUser(db, { id: "reviewer", is_moderator: 1 }); const claimAction = action.startsWith("claim"); await insertDeveloper(db, { - id: "review-developer", + identifier: "review-developer", type: "individual", name: "Developer", owner_user_id: claimAction ? null : "owner" }); await insertExtension(db, { - id: "review-extension", - developer_id: "review-developer" + identifier: "review-extension", + developer_identifier: "review-developer" }); await insertRevision(db, { id: "review-revision", - extension_id: "review-extension", - developer_id: "review-developer", + extension_identifier: "review-extension", + developer_identifier: "review-developer", submitted_by: "owner", content: JSON.stringify(sampleContent({ name: "Reviewed content" })) }); await insertDeveloperClaim(db, { id: "review-claim", - developer_id: "review-developer", + developer_identifier: "review-developer", claimant_id: "claimant" }); await insertDeveloperClaim(db, { id: "competing-claim", - developer_id: "review-developer", + developer_identifier: "review-developer", claimant_id: "competitor" }); const headers = await authHeaders("reviewer"); diff --git a/test/services/extensions/v2/moderation.test.ts b/test/services/extensions/v2/moderation.test.ts index e9f0a303..6d60e6ea 100644 --- a/test/services/extensions/v2/moderation.test.ts +++ b/test/services/extensions/v2/moderation.test.ts @@ -1,3 +1,4 @@ +import { UUID_V7 } from "./db-fixtures"; import { describe, it, expect, vi } from "vitest"; import { env } from "cloudflare:workers"; import { wrapD1WithHook } from "./db-interceptor"; @@ -48,9 +49,9 @@ async function createPending( ); expect(res.status).toBe(201); const { result } = (await res.json()) as { - result: { id: string; revision_id: string }; + result: { id: string; identifier: string; revision_id: string }; }; - return { id: result.id, revisionId: result.revision_id }; + return { id: result.identifier, revisionId: result.revision_id }; } function reviewPath( @@ -100,7 +101,7 @@ describe("Extensions API v2", () => { expect(data.result).toHaveLength(1); expect(data.result[0]).toMatchObject({ status: "pending", - extension_id: "new-ext" + extension_id: expect.stringMatching(UUID_V7) }); }); }); @@ -109,8 +110,8 @@ describe("Extensions API v2", () => { it("serves the public projection to unrelated callers for a published extension", async () => { await seedDeveloper("new-developer", "user-1"); await insertExtension(db, { - id: "live-ext", - developer_id: "new-developer" + identifier: "live-ext", + developer_identifier: "new-developer" }); const res = await get( @@ -119,9 +120,9 @@ describe("Extensions API v2", () => { ); expect(res.status).toBe(200); const data = (await res.json()) as { - result: { id: string; pending_revision?: unknown }; + result: { id: string; identifier: string; pending_revision?: unknown }; }; - expect(data.result.id).toBe("live-ext"); + expect(data.result.identifier).toBe("live-ext"); expect("pending_revision" in data.result).toBe(false); }); @@ -139,8 +140,8 @@ describe("Extensions API v2", () => { await insertUser(db, { id: "mod-1", is_moderator: 1 }); await seedDeveloper("new-developer", "user-1"); await insertExtension(db, { - id: "LIVE-ext", - developer_id: "new-developer" + identifier: "LIVE-ext", + developer_identifier: "new-developer" }); const res = await get( @@ -149,9 +150,13 @@ describe("Extensions API v2", () => { ); expect(res.status).toBe(200); const data = (await res.json()) as { - result: { id: string; published: { name: string } | null }; + result: { + id: string; + identifier: string; + published: { name: string } | null; + }; }; - expect(data.result.id).toBe("LIVE-ext"); + expect(data.result.identifier).toBe("LIVE-ext"); expect(data.result.published?.name).toBe("Extension"); }); @@ -159,8 +164,8 @@ describe("Extensions API v2", () => { await insertUser(db, { id: "mod-1", is_moderator: 1 }); await seedDeveloper("new-developer", "user-1"); await insertExtension(db, { - id: "live-ext", - developer_id: "new-developer" + identifier: "live-ext", + developer_identifier: "new-developer" }); await post( "/extensions/v2/extensions/live-ext/delist", @@ -193,12 +198,12 @@ describe("Extensions API v2", () => { await insertUser(db, { id: "mod-1", is_moderator: 1 }); await seedDeveloper("new-developer", "user-1"); await insertExtension(db, { - id: "live-ext", - developer_id: "new-developer" + identifier: "live-ext", + developer_identifier: "new-developer" }); await insertUnpublishedExtension(db, { - id: "draft-ext", - developer_id: "new-developer" + identifier: "draft-ext", + developer_identifier: "new-developer" }); await post( "/extensions/v2/extensions/live-ext/delist", @@ -207,8 +212,8 @@ describe("Extensions API v2", () => { ); await seedDeveloper("other-developer", "user-2"); await insertExtension(db, { - id: "other-ext", - developer_id: "other-developer" + identifier: "other-ext", + developer_identifier: "other-developer" }); const res = await get( @@ -216,8 +221,10 @@ describe("Extensions API v2", () => { await authHeaders("mod-1") ); expect(res.status).toBe(200); - const body = (await res.json()) as { result: Array<{ id: string }> }; - expect(body.result.map((r) => r.id)).toEqual([ + const body = (await res.json()) as { + result: Array<{ id: string; identifier: string }>; + }; + expect(body.result.map((r) => r.identifier)).toEqual([ "draft-ext", "live-ext", "other-ext" @@ -228,16 +235,16 @@ describe("Extensions API v2", () => { await insertUser(db, { id: "mod-1", is_moderator: 1 }); await seedDeveloper("new-developer", "user-1"); await insertExtension(db, { - id: "live-ext", - developer_id: "new-developer" + identifier: "live-ext", + developer_identifier: "new-developer" }); await insertExtension(db, { - id: "delisted-ext", - developer_id: "new-developer" + identifier: "delisted-ext", + developer_identifier: "new-developer" }); await insertUnpublishedExtension(db, { - id: "draft-ext", - developer_id: "new-developer" + identifier: "draft-ext", + developer_identifier: "new-developer" }); await post( "/extensions/v2/extensions/delisted-ext/delist", @@ -250,7 +257,7 @@ describe("Extensions API v2", () => { await authHeaders("mod-1") ); expect(await published.json()).toMatchObject({ - result: [{ id: "live-ext" }] + result: [{ id: expect.stringMatching(UUID_V7), identifier: "live-ext" }] }); const delisted = await get( @@ -258,7 +265,9 @@ describe("Extensions API v2", () => { await authHeaders("mod-1") ); expect(await delisted.json()).toMatchObject({ - result: [{ id: "delisted-ext" }] + result: [ + { id: expect.stringMatching(UUID_V7), identifier: "delisted-ext" } + ] }); const unpublished = await get( @@ -266,7 +275,9 @@ describe("Extensions API v2", () => { await authHeaders("mod-1") ); expect(await unpublished.json()).toMatchObject({ - result: [{ id: "draft-ext" }] + result: [ + { id: expect.stringMatching(UUID_V7), identifier: "draft-ext" } + ] }); }); @@ -274,16 +285,16 @@ describe("Extensions API v2", () => { await insertUser(db, { id: "mod-1", is_moderator: 1 }); await seedDeveloper("new-developer", "user-1"); await insertExtension(db, { - id: "paygate", - developer_id: "new-developer" + identifier: "paygate", + developer_identifier: "new-developer" }); await insertExtension(db, { - id: "other-gateway", - developer_id: "new-developer" + identifier: "other-gateway", + developer_identifier: "new-developer" }); await insertExtension(db, { - id: "unrelated", - developer_id: "new-developer" + identifier: "unrelated", + developer_identifier: "new-developer" }); const res = await get( @@ -291,8 +302,10 @@ describe("Extensions API v2", () => { await authHeaders("mod-1") ); expect(res.status).toBe(200); - const body = (await res.json()) as { result: Array<{ id: string }> }; - expect(body.result.map((r) => r.id).sort()).toEqual([ + const body = (await res.json()) as { + result: Array<{ id: string; identifier: string }>; + }; + expect(body.result.map((r) => r.identifier).sort()).toEqual([ "other-gateway", "paygate" ]); @@ -302,8 +315,8 @@ describe("Extensions API v2", () => { await insertUser(db, { id: "mod-1", is_moderator: 1 }); await seedDeveloper("new-developer", "user-1"); await insertExtension(db, { - id: "pay-gate", - developer_id: "new-developer" + identifier: "pay-gate", + developer_identifier: "new-developer" }); const percentRes = await get( @@ -432,7 +445,11 @@ describe("Extensions API v2", () => { const v1Res = await get("/extensions/v1/new-ext", {}); expect(v1Res.status).toBe(200); const v1Body = (await v1Res.json()) as { - result: { id: string; author: { id: string } }; + result: { + id: string; + identifier: string; + author: { id: string; identifier: string }; + }; }; expect(v1Body.result.id).toBe("new-ext"); expect(v1Body.result.author.id).toBe("new-developer"); @@ -538,13 +555,13 @@ describe("Extensions API v2", () => { await insertUser(db, { id: "mod-1", is_moderator: 1 }); await seedDeveloper("new-developer", "user-1"); await insertUnpublishedExtension(db, { - id: "legacy-ext", - developer_id: "new-developer" + identifier: "legacy-ext", + developer_identifier: "new-developer" }); await insertRevision(db, { id: "legacy-revision", - extension_id: "legacy-ext", - developer_id: "new-developer", + extension_identifier: "legacy-ext", + developer_identifier: "new-developer", submitted_by: "user-1", // Carried through by migration 0021 from a submission written under // rules that required far less than today's. @@ -689,15 +706,15 @@ describe("Extensions API v2", () => { // cannot fork a second, lowercase one. it("edits a mixed-case legacy extension in place", async () => { await insertDeveloper(db, { - id: "owner-developer", + identifier: "owner-developer", type: "user", name: "Owner", url: null, owner_user_id: "owner-1" }); await insertExtension(db, { - id: "Existing-Ext", - developer_id: "owner-developer", + identifier: "Existing-Ext", + developer_identifier: "owner-developer", name: "Existing" }); await insertUser(db, { id: "mod-1", is_moderator: 1 }); @@ -840,8 +857,8 @@ describe("Extensions API v2", () => { it("requires moderator access", async () => { await seedDeveloper("new-developer", "user-1"); await insertExtension(db, { - id: "live-ext", - developer_id: "new-developer" + identifier: "live-ext", + developer_identifier: "new-developer" }); const res = await post( @@ -867,8 +884,8 @@ describe("Extensions API v2", () => { await insertUser(db, { id: "mod-1", is_moderator: 1 }); await seedDeveloper("new-developer", "user-1"); await insertUnpublishedExtension(db, { - id: "draft-ext", - developer_id: "new-developer" + identifier: "draft-ext", + developer_identifier: "new-developer" }); const res = await post( @@ -883,8 +900,8 @@ describe("Extensions API v2", () => { await insertUser(db, { id: "mod-1", is_moderator: 1 }); await seedDeveloper("new-developer", "user-1"); await insertExtension(db, { - id: "live-ext", - developer_id: "new-developer" + identifier: "live-ext", + developer_identifier: "new-developer" }); const res = await post( @@ -899,8 +916,8 @@ describe("Extensions API v2", () => { await insertUser(db, { id: "mod-1", is_moderator: 1 }); await seedDeveloper("new-developer", "user-1"); await insertExtension(db, { - id: "live-ext", - developer_id: "new-developer" + identifier: "live-ext", + developer_identifier: "new-developer" }); const res = await post( @@ -915,8 +932,8 @@ describe("Extensions API v2", () => { await insertUser(db, { id: "mod-1", is_moderator: 1 }); await seedDeveloper("new-developer", "user-1"); await insertExtension(db, { - id: "LIVE-ext", - developer_id: "new-developer", + identifier: "LIVE-ext", + developer_identifier: "new-developer", updated_at: "2020-01-01T00:00:00.000Z" }); @@ -927,7 +944,12 @@ describe("Extensions API v2", () => { ); expect(res.status).toBe(200); await expect(res.json()).resolves.toEqual({ - result: { id: "live-ext", status: "delisted", notified: false } + result: { + id: expect.stringMatching(UUID_V7), + identifier: "LIVE-ext", + status: "delisted", + notified: false + } }); // Case-insensitively addressed, like every other extension route. @@ -963,8 +985,8 @@ describe("Extensions API v2", () => { await insertUser(db, { id: "mod-1", is_moderator: 1 }); await seedDeveloper("new-developer", "user-1"); await insertExtension(db, { - id: "live-ext", - developer_id: "new-developer" + identifier: "live-ext", + developer_identifier: "new-developer" }); const mod = await authHeaders("mod-1"); await post("/extensions/v2/extensions/live-ext/delist", mod, { @@ -989,8 +1011,8 @@ describe("Extensions API v2", () => { await insertUser(db, { id: "mod-1", is_moderator: 1 }); await seedDeveloper("new-developer", "user-1"); await insertExtension(db, { - id: "live-ext", - developer_id: "new-developer" + identifier: "live-ext", + developer_identifier: "new-developer" }); await post( "/extensions/v2/extensions/live-ext/delist", @@ -1041,8 +1063,8 @@ describe("Extensions API v2", () => { it("requires moderator access", async () => { await seedDeveloper("new-developer", "user-1"); await insertExtension(db, { - id: "live-ext", - developer_id: "new-developer" + identifier: "live-ext", + developer_identifier: "new-developer" }); const res = await post( @@ -1068,8 +1090,8 @@ describe("Extensions API v2", () => { await insertUser(db, { id: "mod-1", is_moderator: 1 }); await seedDeveloper("new-developer", "user-1"); await insertExtension(db, { - id: "live-ext", - developer_id: "new-developer" + identifier: "live-ext", + developer_identifier: "new-developer" }); const res = await post( @@ -1089,8 +1111,8 @@ describe("Extensions API v2", () => { await insertUser(db, { id: "mod-1", is_moderator: 1 }); await seedDeveloper("new-developer", "user-1"); await insertExtension(db, { - id: "live-ext", - developer_id: "new-developer" + identifier: "live-ext", + developer_identifier: "new-developer" }); const mod = await authHeaders("mod-1"); await post("/extensions/v2/extensions/live-ext/delist", mod, { @@ -1104,7 +1126,12 @@ describe("Extensions API v2", () => { ); expect(res.status).toBe(200); await expect(res.json()).resolves.toEqual({ - result: { id: "live-ext", status: "relisted", notified: false } + result: { + id: expect.stringMatching(UUID_V7), + identifier: "live-ext", + status: "relisted", + notified: false + } }); }); @@ -1112,8 +1139,8 @@ describe("Extensions API v2", () => { await insertUser(db, { id: "mod-1", is_moderator: 1 }); await seedDeveloper("new-developer", "user-1"); await insertExtension(db, { - id: "live-ext", - developer_id: "new-developer" + identifier: "live-ext", + developer_identifier: "new-developer" }); const mod = await authHeaders("mod-1"); await post("/extensions/v2/extensions/live-ext/delist", mod, { @@ -1129,7 +1156,12 @@ describe("Extensions API v2", () => { }); expect(res.status).toBe(200); await expect(res.json()).resolves.toEqual({ - result: { id: "live-ext", status: "relisted", notified: false } + result: { + id: expect.stringMatching(UUID_V7), + identifier: "live-ext", + status: "relisted", + notified: false + } }); expect(await getExtension(db, "live-ext")).toMatchObject({ @@ -1137,15 +1169,17 @@ describe("Extensions API v2", () => { }); await expect( (await get("/extensions/v2/extensions", {})).json() - ).resolves.toMatchObject({ result: [{ id: "live-ext" }] }); + ).resolves.toMatchObject({ + result: [{ id: expect.stringMatching(UUID_V7), identifier: "live-ext" }] + }); }); it("answers with the stored canonical id for a mixed-case path", async () => { await insertUser(db, { id: "mod-1", is_moderator: 1 }); await seedDeveloper("new-developer", "user-1"); await insertExtension(db, { - id: "LIVE-ext", - developer_id: "new-developer" + identifier: "LIVE-ext", + developer_identifier: "new-developer" }); const mod = await authHeaders("mod-1"); await post("/extensions/v2/extensions/live-ext/delist", mod, { @@ -1159,7 +1193,12 @@ describe("Extensions API v2", () => { ); expect(res.status).toBe(200); await expect(res.json()).resolves.toEqual({ - result: { id: "LIVE-ext", status: "relisted", notified: false } + result: { + id: expect.stringMatching(UUID_V7), + identifier: "LIVE-ext", + status: "relisted", + notified: false + } }); }); @@ -1167,8 +1206,8 @@ describe("Extensions API v2", () => { await insertUser(db, { id: "mod-1", is_moderator: 1 }); await seedDeveloper("new-developer", "user-1"); await insertExtension(db, { - id: "live-ext", - developer_id: "new-developer" + identifier: "live-ext", + developer_identifier: "new-developer" }); const mod = await authHeaders("mod-1"); await post("/extensions/v2/extensions/live-ext/delist", mod, { @@ -1301,10 +1340,11 @@ describe("Extensions API v2", () => { ); expect(approve.status).toBe(200); const approveBody = (await approve.json()) as { - result: { id: string; approved: boolean }; + result: { id: string; identifier: string; approved: boolean }; }; expect(approveBody.result).toEqual({ - id: "dev-developer", + id: expect.stringMatching(UUID_V7), + identifier: "dev-developer", approved: true, notified: false }); @@ -1315,9 +1355,9 @@ describe("Extensions API v2", () => { ); expect(unapproved.status).toBe(200); const unapprovedBody = (await unapproved.json()) as { - result: Array<{ id: string }>; + result: Array<{ id: string; identifier: string }>; }; - expect(unapprovedBody.result.map((a) => a.id)).not.toContain( + expect(unapprovedBody.result.map((a) => a.identifier)).not.toContain( "dev-developer" ); }); @@ -1365,17 +1405,17 @@ describe("Extensions API v2", () => { ); expect(res.status).toBe(200); const body = (await res.json()) as { - result: Array<{ id: string; approved: boolean }>; + result: Array<{ id: string; identifier: string; approved: boolean }>; }; - expect(body.result.map((a) => a.id).sort()).toEqual([ + expect(body.result.map((a) => a.identifier).sort()).toEqual([ "dev-developer", "other-developer" ]); - expect(body.result.find((a) => a.id === "dev-developer")?.approved).toBe( - true - ); expect( - body.result.find((a) => a.id === "other-developer")?.approved + body.result.find((a) => a.identifier === "dev-developer")?.approved + ).toBe(true); + expect( + body.result.find((a) => a.identifier === "other-developer")?.approved ).toBe(false); }); @@ -1402,7 +1442,9 @@ describe("Extensions API v2", () => { ); expect(scoped.status).toBe(200); await expect(scoped.json()).resolves.toMatchObject({ - result: [{ id: "dev-developer" }], + result: [ + { id: expect.stringMatching(UUID_V7), identifier: "dev-developer" } + ], pagination: { next_cursor: null, has_more: false } }); @@ -1430,10 +1472,12 @@ describe("Extensions API v2", () => { const first = await get("/extensions/v2/developers?limit=1", mod); expect(first.status).toBe(200); const firstBody = (await first.json()) as { - result: Array<{ id: string }>; + result: Array<{ id: string; identifier: string }>; pagination: { next_cursor: string | null; has_more: boolean }; }; - expect(firstBody.result.map((d) => d.id)).toEqual(["aaa-developer"]); + expect(firstBody.result.map((d) => d.identifier)).toEqual([ + "aaa-developer" + ]); expect(firstBody.pagination.has_more).toBe(true); expect(firstBody.pagination.next_cursor).toBeTruthy(); @@ -1443,10 +1487,12 @@ describe("Extensions API v2", () => { ); expect(second.status).toBe(200); const secondBody = (await second.json()) as { - result: Array<{ id: string }>; + result: Array<{ id: string; identifier: string }>; pagination: { next_cursor: string | null; has_more: boolean }; }; - expect(secondBody.result.map((d) => d.id)).toEqual(["zzz-developer"]); + expect(secondBody.result.map((d) => d.identifier)).toEqual([ + "zzz-developer" + ]); expect(secondBody.pagination).toEqual({ next_cursor: null, has_more: false @@ -1498,7 +1544,9 @@ describe("Extensions API v2", () => { }>; }; expect(data.result).toHaveLength(1); - expect(data.result[0].developer_id).toBe("dev-developer"); + expect(data.result[0].developer_id).toBe( + (await getDeveloper(db, "dev-developer"))!.id + ); expect(data.result[0].name).toBe("Dev Developer"); expect(data.result[0].changed_by).toBe("user-1"); }); @@ -1671,31 +1719,31 @@ describe("Extensions API v2", () => { await insertUser(db, { id: "mod-1", is_moderator: 1 }); await seedDeveloper("new-developer", "user-1"); await insertExtension(db, { - id: "live-ext", - developer_id: "new-developer" + identifier: "live-ext", + developer_identifier: "new-developer" }); await insertExtension(db, { - id: "gone-ext", - developer_id: "new-developer", + identifier: "gone-ext", + developer_identifier: "new-developer", delisted_at: new Date().toISOString(), delist_reason: "Upstream source removed" }); await insertUnpublishedExtension(db, { - id: "draft-ext", - developer_id: "new-developer" + identifier: "draft-ext", + developer_identifier: "new-developer" }); const content = JSON.stringify(sampleContent()); await insertRevision(db, { id: "pending-revision", - extension_id: "live-ext", - developer_id: "new-developer", + extension_identifier: "live-ext", + developer_identifier: "new-developer", submitted_by: "user-1", content }); await insertRevision(db, { id: "approved-revision", - extension_id: "live-ext", - developer_id: "new-developer", + extension_identifier: "live-ext", + developer_identifier: "new-developer", submitted_by: "user-1", status: "approved", reviewer_id: "mod-1", @@ -1703,8 +1751,8 @@ describe("Extensions API v2", () => { }); await insertRevision(db, { id: "rejected-revision", - extension_id: "live-ext", - developer_id: "new-developer", + extension_identifier: "live-ext", + developer_identifier: "new-developer", submitted_by: "user-1", status: "rejected", reviewer_id: "mod-1", diff --git a/test/services/extensions/v2/ownership.test.ts b/test/services/extensions/v2/ownership.test.ts index a7ee40ea..85bcb915 100644 --- a/test/services/extensions/v2/ownership.test.ts +++ b/test/services/extensions/v2/ownership.test.ts @@ -105,9 +105,9 @@ describe("Extensions API v2", () => { ); expect(accept.status).toBe(200); const accepted = (await accept.json()) as { - result: { id: string; approved: boolean }; + result: { id: string; identifier: string; approved: boolean }; }; - expect(accepted.result.id).toBe("dev-developer"); + expect(accepted.result.identifier).toBe("dev-developer"); expect(accepted.result.approved).toBe(false); expect((await getDeveloper(db, "dev-developer"))?.owner_user_id).toBe( "user-2" @@ -146,11 +146,13 @@ describe("Extensions API v2", () => { ) { deleted = true; await db - .prepare("DELETE FROM developer_transfers WHERE developer_id = ?") + .prepare( + "DELETE FROM developer_transfers WHERE developer_id = (SELECT id FROM developers WHERE identifier=?)" + ) .bind("dev-developer") .run(); await db - .prepare("DELETE FROM developers WHERE id = ?") + .prepare("DELETE FROM developers WHERE identifier = ?") .bind("dev-developer") .run(); } @@ -178,17 +180,17 @@ describe("Extensions API v2", () => { ); await insertDeveloperClaim(db, { id: "transfer-pending-claim", - developer_id: "dev-developer", + developer_identifier: "dev-developer", claimant_id: "user-3" }); await insertUnpublishedExtension(db, { - id: "transfer-pending-ext", - developer_id: "dev-developer" + identifier: "transfer-pending-ext", + developer_identifier: "dev-developer" }); await insertRevision(db, { id: "transfer-pending-revision", - extension_id: "transfer-pending-ext", - developer_id: "dev-developer", + extension_identifier: "transfer-pending-ext", + developer_identifier: "dev-developer", submitted_by: "user-3", content: JSON.stringify(sampleContent()) }); @@ -573,6 +575,19 @@ describe("Extensions API v2", () => { }); describe("developer claims", () => { + it("does not treat an internal UUID as a public identifier", async () => { + await seedUnownedDeveloper("legacy-developer"); + const developer = (await getDeveloper(db, "legacy-developer"))!; + const response = await post( + `/extensions/v2/developers/${developer.id}/claim`, + await authHeaders("claimant"), + {} + ); + expect(response.status).toBe(404); + expect(await countDeveloperClaims(db)).toBe(0); + expect(ghRequest).not.toHaveBeenCalled(); + }); + // The claim body is strict so the server-computed verification fields on // DeveloperClaimSchema can never be supplied by the caller. Unknown keys // are reported at the root path, since the body has no nesting. @@ -695,7 +710,7 @@ describe("Extensions API v2", () => { github_orgs_expires_at }); await insertDeveloper(db, { - id: "acme-org", + identifier: "acme-org", type: "organization", name: "Acme Org", url: null, @@ -886,7 +901,7 @@ describe("Extensions API v2", () => { // transition therefore executes first and this NOT NULL violation // proves D1 rolls it back. return db.prepare( - "UPDATE developers SET name = NULL WHERE id = 'legacy-developer'" + "UPDATE developers SET name = NULL WHERE identifier = 'legacy-developer'" ); } }); @@ -931,9 +946,9 @@ describe("Extensions API v2", () => { ); expect(approve.status).toBe(200); const approved = (await approve.json()) as { - result: { id: string; approved: boolean }; + result: { id: string; identifier: string; approved: boolean }; }; - expect(approved.result.id).toBe("legacy-developer"); + expect(approved.result.identifier).toBe("legacy-developer"); expect(approved.result.approved).toBe(false); expect((await getDeveloper(db, "legacy-developer"))?.owner_user_id).toBe( "user-1" @@ -983,7 +998,7 @@ describe("Extensions API v2", () => { it("copies the claim's GitHub verification onto the developer row it transfers ownership to", async () => { await insertDeveloper(db, { - id: "legacy-developer", + identifier: "legacy-developer", type: "organization", name: "Legacy Developer", url: null, @@ -1073,7 +1088,7 @@ describe("Extensions API v2", () => { it("verifies a claim when the claimant's linked GitHub org matches the developer id", async () => { await insertDeveloper(db, { - id: "legacy-developer", + identifier: "legacy-developer", type: "organization", name: "Legacy Developer", url: null, @@ -1100,7 +1115,7 @@ describe("Extensions API v2", () => { it("verifies a claim when the claimant's linked GitHub login matches a user-type developer id", async () => { await insertDeveloper(db, { - id: "legacy-user", + identifier: "legacy-user", type: "user", name: "Legacy User", url: null, @@ -1127,7 +1142,7 @@ describe("Extensions API v2", () => { it("blocks a claim outright when the claimant's linked GitHub identity doesn't match", async () => { await insertDeveloper(db, { - id: "legacy-developer", + identifier: "legacy-developer", type: "organization", name: "Legacy Developer", url: null, @@ -1170,7 +1185,7 @@ describe("Extensions API v2", () => { it("falls back to unverified manual review when organization membership evidence is stale", async () => { await insertDeveloper(db, { - id: "legacy-developer", + identifier: "legacy-developer", type: "organization", name: "Legacy Developer", url: null, @@ -1198,7 +1213,7 @@ describe("Extensions API v2", () => { it("does not verify an organization claim for a whitespace-only GitHub login", async () => { await insertDeveloper(db, { - id: "legacy-developer", + identifier: "legacy-developer", type: "organization", name: "Legacy Developer", url: null, diff --git a/test/services/extensions/v2/packages.test.ts b/test/services/extensions/v2/packages.test.ts index b4d0ca2d..dce38b81 100644 --- a/test/services/extensions/v2/packages.test.ts +++ b/test/services/extensions/v2/packages.test.ts @@ -42,7 +42,7 @@ import { put, db } from "./harness"; -import { insertUser, getExtension } from "./db-fixtures"; +import { insertUser, getExtension, getDeveloper } from "./db-fixtures"; vi.mock("@octokit/request", async () => (await import("../../../mocks/octokit")).octokitRequestMock() @@ -78,8 +78,7 @@ afterEach(() => { "S3_BUCKET", "S3_ACCESS_KEY_ID", "S3_SECRET_ACCESS_KEY", - "PUBLIC_URL", - "S3_PREFIX" + "PUBLIC_URL" ]) delete (env as unknown as Record)[ `EXTENSION_ASSETS_${key}` @@ -198,7 +197,7 @@ async function submitted( const response = await post( `${root}/extensions`, await authHeaders("owner"), - { id: "package-demo", ...content(artifact) } + { identifier: "package-demo", ...content(artifact) } ); const body = (await response.json()) as { result: { revision_id: string } }; expect(response.status, JSON.stringify(body)).toBe(201); @@ -232,7 +231,10 @@ describe("publisher package builds", () => { const response = await put( `${root}/m2m/clients/package-publisher`, await authHeaders("publisher-moderator"), - { developer_id: "developer", can_publish: true } + { + developer_id: (await getDeveloper(db, "developer"))!.id, + can_publish: true + } ); expect(response.status).toBe(200); }); @@ -319,7 +321,7 @@ describe("publisher package builds", () => { "Content-Type": "application/json" }; const created = await post(`${root}/extensions`, headers, { - id: "package-demo", + identifier: "package-demo", ...content(artifact) }); expect(created.status).toBe(201); @@ -358,7 +360,7 @@ describe("publisher package builds", () => { await seedDeveloper("another-developer", "another-owner"); await db .prepare( - "UPDATE m2m_publisher_clients SET developer_id='another-developer' WHERE client_id='package-publisher'" + "UPDATE m2m_publisher_clients SET developer_id=(SELECT id FROM developers WHERE identifier='another-developer') WHERE client_id='package-publisher'" ) .run(); // New IDs can be reserved by that developer, but an existing owned ID cannot. @@ -454,7 +456,7 @@ describe("queued package preparation", () => { }; await db .prepare( - "UPDATE developers SET ownership_epoch=ownership_epoch+1 WHERE id='developer'" + "UPDATE developers SET ownership_epoch=ownership_epoch+1 WHERE identifier = 'developer'" ) .run(); expect( @@ -708,7 +710,7 @@ describe("stored directory packages", () => { }); await db .prepare( - "UPDATE extensions SET delisted_at=CURRENT_TIMESTAMP WHERE id='package-demo'" + "UPDATE extensions SET delisted_at=CURRENT_TIMESTAMP WHERE identifier='package-demo'" ) .run(); expect((await get(path, {})).status).toBe(404); @@ -771,7 +773,7 @@ describe("stored directory packages", () => { ); await db .prepare( - "UPDATE extensions SET delisted_at=CURRENT_TIMESTAMP WHERE id='package-demo'" + "UPDATE extensions SET delisted_at=CURRENT_TIMESTAMP WHERE identifier='package-demo'" ) .run(); expect((await get(path, {})).status).toBe(404); @@ -977,8 +979,7 @@ describe("stored directory packages", () => { EXTENSION_ASSETS_S3_BUCKET: "test-assets", EXTENSION_ASSETS_S3_ACCESS_KEY_ID: "test-key", EXTENSION_ASSETS_S3_SECRET_ACCESS_KEY: "test-secret", - EXTENSION_ASSETS_PUBLIC_URL: "https://extensions-assets.fossbilling.net", - EXTENSION_ASSETS_S3_PREFIX: `tests-${crypto.randomUUID()}` + EXTENSION_ASSETS_PUBLIC_URL: "https://extensions-assets.fossbilling.net" }); const calls: Request[] = []; let failure = false; @@ -1025,7 +1026,7 @@ describe("stored directory packages", () => { ); return writer.close(); } - it("keeps pending images private and publishes approved images to B2 only once", async () => { + it("keeps pending images private and publishes a stable UUID icon URL", async () => { const remote = configureB2(); const artifact = await built(manifest(), await iconSource()); const pending = await submitted(artifact); @@ -1040,14 +1041,16 @@ describe("stored directory packages", () => { expect(remote.calls).toHaveLength(0); expect((await approve(pending)).status).toBe(200); const published = await getExtension(db, "package-demo"); - expect(published?.icon_url).toMatch( - /^https:\/\/extensions-assets\.fossbilling\.net\/tests-[a-f0-9-]+\/icons\/[a-f0-9]{64}\.svg$/ + const objectKey = `extensions/${published!.id}/icons/${artifact.source_id}.svg`; + expect(published?.icon_url).toBe( + `https://extensions-assets.fossbilling.net/${objectKey}` ); expect(remote.calls.map((request) => request.method)).toEqual([ "HEAD", "PUT" ]); const upload = remote.calls[1]; + expect(new URL(upload.url).pathname).toBe(`/test-assets/${objectKey}`); expect(upload.headers.get("Authorization")).toContain( "AWS4-HMAC-SHA256 Credential=test-key/" ); @@ -1060,6 +1063,25 @@ describe("stored directory packages", () => { ); await get(`${root}/packages/${artifact.id}/icon`, {}); expect(remote.calls).toHaveLength(2); + const edited = await built( + { ...manifest(), description: "Updated metadata" }, + undefined, + artifact.source_id + ); + const update = await put( + `${root}/extensions/package-demo`, + await authHeaders("owner"), + content(edited) + ); + expect(update.status).toBe(202); + const { result } = (await update.json()) as { + result: { revision_id: string }; + }; + expect((await approve(result.revision_id)).status).toBe(200); + expect((await getExtension(db, "package-demo"))?.icon_url).toBe( + published!.icon_url + ); + expect(remote.calls).toHaveLength(2); }); it("keeps approval and fallback delivery working during B2 failures, then retries publication", async () => { const remote = configureB2(); @@ -1080,7 +1102,7 @@ describe("stored directory packages", () => { 200 ); expect((await getExtension(db, "package-demo"))?.icon_url).toContain( - "https://extensions-assets.fossbilling.net/tests-" + "https://extensions-assets.fossbilling.net/extensions/" ); }); it("publishes an icon when a valid extension ID is supplied in uppercase", async () => { @@ -1112,6 +1134,10 @@ describe("stored directory packages", () => { expect((await approve(await submitted(artifact))).status).toBe(200); const uploads = remote.calls.filter((request) => request.method === "PUT"); expect(uploads).toHaveLength(2); + const extension = (await getExtension(db, "package-demo"))!; + expect(new URL(uploads[0].url).pathname).toBe( + `/test-assets/extensions/${extension.id}/icons/${artifact.source_id}.avif` + ); expect(uploads[0].url.replace(/\.avif$/, ".png")).toBe(uploads[1].url); expect( uploads.map((request) => request.headers.get("Content-Type")) @@ -1212,14 +1238,14 @@ describe("stored directory packages", () => { const response = await post( `${root}/extensions`, await authHeaders("owner"), - { id: "package-demo", ...content(artifact), ...patch } + { identifier: "package-demo", ...content(artifact), ...patch } ); expect(response.status).toBe(409); } expect( await db .prepare( - "SELECT count(*) AS count FROM extensions WHERE id='package-demo'" + "SELECT count(*) AS count FROM extensions WHERE identifier='package-demo'" ) .first() ).toEqual({ count: 0 }); @@ -1461,7 +1487,7 @@ describe("stored directory packages", () => { `${root}/extensions`, await authHeaders("owner"), { - id: "package-demo", + identifier: "package-demo", ...payload, releases: payload.releases.map((release) => ({ ...release, @@ -1473,7 +1499,7 @@ describe("stored directory packages", () => { const response = await post( `${root}/extensions`, await authHeaders("owner"), - { id: "package-demo", ...payload } + { identifier: "package-demo", ...payload } ); expect(response.status).toBe(201); expect( diff --git a/test/services/extensions/v2/pending-edits.test.ts b/test/services/extensions/v2/pending-edits.test.ts index 41c53f6c..59e89aa7 100644 --- a/test/services/extensions/v2/pending-edits.test.ts +++ b/test/services/extensions/v2/pending-edits.test.ts @@ -75,13 +75,13 @@ describe("pending submission replacement", () => { it("lets the owner replace a publisher submission and attributes the replacement to the owner", async () => { await seedDeveloper("dev", "owner"); await insertUnpublishedExtension(db, { - id: "new-ext", - developer_id: "dev" + identifier: "new-ext", + developer_identifier: "dev" }); await insertRevision(db, { id: "publisher-revision", - extension_id: "new-ext", - developer_id: "dev", + extension_identifier: "new-ext", + developer_identifier: "dev", submitted_by: "publisher-client", content: JSON.stringify(sampleContent()) }); @@ -107,13 +107,13 @@ describe("pending submission replacement", () => { async (extensionId) => { await seedDeveloper("dev", "owner"); await insertUnpublishedExtension(db, { - id: extensionId, - developer_id: "dev" + identifier: extensionId, + developer_identifier: "dev" }); await insertRevision(db, { id: "canonical-revision", - extension_id: extensionId, - developer_id: "dev", + extension_identifier: extensionId, + developer_identifier: "dev", submitted_by: "owner", content: JSON.stringify(sampleContent()) }); @@ -124,11 +124,11 @@ describe("pending submission replacement", () => { ); expect(response.status).toBe(202); const { result } = (await response.json()) as { - result: { id: string; revision_id: string }; + result: { id: string; identifier: string; revision_id: string }; }; - expect(result.id).toBe(extensionId); + expect(result.identifier).toBe(extensionId); expect((await getRevision(db, result.revision_id))!.extension_id).toBe( - result.id + (await getExtension(db, extensionId))!.id ); } ); diff --git a/test/services/extensions/v2/public-extensions.test.ts b/test/services/extensions/v2/public-extensions.test.ts index 0c3e4302..7f784b70 100644 --- a/test/services/extensions/v2/public-extensions.test.ts +++ b/test/services/extensions/v2/public-extensions.test.ts @@ -1,3 +1,4 @@ +import { UUID_V7 } from "./db-fixtures"; import { describe, it, expect, vi } from "vitest"; import { setupExtensionsV2Tests, db, get, seedOwnedExtension } from "./harness"; import { insertDeveloper, insertExtension } from "./db-fixtures"; @@ -14,7 +15,7 @@ describe("Extensions API v2", () => { describe("GET /extensions", () => { async function seedExtensions(ids: string[]): Promise { await insertDeveloper(db, { - id: "listing-developer", + identifier: "listing-developer", type: "user", name: "Listing Developer", url: null, @@ -22,9 +23,9 @@ describe("Extensions API v2", () => { }); for (const id of ids) { await insertExtension(db, { - id, + identifier: id, type: "module", - developer_id: "listing-developer", + developer_identifier: "listing-developer", name: id, description: `Description for ${id}`, releases: '[{"tag":"1.0.0"}]', @@ -47,12 +48,13 @@ describe("Extensions API v2", () => { const body = (await res.json()) as { result: Array<{ id: string; - developer: { id: string; unclaimed: boolean }; + identifier: string; + developer: { id: string; identifier: string; unclaimed: boolean }; }>; }; expect(body.result).toHaveLength(1); - expect(body.result[0].id).toBe("existing-ext"); - expect(body.result[0].developer.id).toBe("owner-developer"); + expect(body.result[0].identifier).toBe("existing-ext"); + expect(body.result[0].developer.identifier).toBe("owner-developer"); expect(body.result[0].developer.unclaimed).toBe(false); expect(body.result[0]).not.toHaveProperty("readme"); expect(body.result[0]).not.toHaveProperty("releases"); @@ -94,14 +96,14 @@ describe("Extensions API v2", () => { await seedOwnedExtension(); const matching = await get( - "/extensions/v2/extensions?developer_id=owner-developer", + "/extensions/v2/extensions?developer_identifier=owner-developer", {} ); const matchingBody = (await matching.json()) as { result: unknown[] }; expect(matchingBody.result).toHaveLength(1); const nonMatching = await get( - "/extensions/v2/extensions?developer_id=someone-else", + "/extensions/v2/extensions?developer_identifier=someone-else", {} ); const nonMatchingBody = (await nonMatching.json()) as { @@ -115,10 +117,13 @@ describe("Extensions API v2", () => { const first = await get("/extensions/v2/extensions?limit=2", {}); const firstBody = (await first.json()) as { - result: Array<{ id: string }>; + result: Array<{ id: string; identifier: string }>; pagination: { next_cursor: string | null; has_more: boolean }; }; - expect(firstBody.result.map(({ id }) => id)).toEqual(["Alpha", "bravo"]); + expect(firstBody.result.map(({ identifier }) => identifier)).toEqual([ + "Alpha", + "bravo" + ]); expect(firstBody.pagination.has_more).toBe(true); const middle = await get( @@ -126,7 +131,7 @@ describe("Extensions API v2", () => { {} ); const middleBody = (await middle.json()) as typeof firstBody; - expect(middleBody.result.map(({ id }) => id)).toEqual([ + expect(middleBody.result.map(({ identifier }) => identifier)).toEqual([ "charlie", "delta" ]); @@ -137,7 +142,9 @@ describe("Extensions API v2", () => { {} ); const finalBody = (await final.json()) as typeof firstBody; - expect(finalBody.result.map(({ id }) => id)).toEqual(["echo"]); + expect(finalBody.result.map(({ identifier }) => identifier)).toEqual([ + "echo" + ]); expect(finalBody.pagination).toEqual({ next_cursor: null, has_more: false @@ -174,7 +181,7 @@ describe("Extensions API v2", () => { ); expect(second.status).toBe(200); expect(await second.json()).toMatchObject({ - result: [{ id: "😀" }], + result: [{ id: expect.stringMatching(UUID_V7), identifier: "😀" }], pagination: { next_cursor: null, has_more: false } }); }); @@ -199,10 +206,11 @@ describe("Extensions API v2", () => { const body = (await res.json()) as { result: { id: string; + identifier: string; developer: { name: string; approved: boolean; unclaimed: boolean }; }; }; - expect(body.result.id).toBe("existing-ext"); + expect(body.result.identifier).toBe("existing-ext"); expect(body.result.developer.name).toBe("Owner"); expect(body.result.developer.approved).toBe(false); expect(body.result.developer.unclaimed).toBe(false); @@ -224,15 +232,15 @@ describe("Extensions API v2", () => { it("404s for a delisted extension, even though it is still published", async () => { await insertDeveloper(db, { - id: "listing-developer", + identifier: "listing-developer", type: "user", name: "Listing Developer", url: null, owner_user_id: null }); await insertExtension(db, { - id: "delisted-ext", - developer_id: "listing-developer", + identifier: "delisted-ext", + developer_identifier: "listing-developer", delisted_at: "2026-01-01T00:00:00.000Z", delist_reason: "Upstream source removed" }); diff --git a/test/services/extensions/v2/publisher.test.ts b/test/services/extensions/v2/publisher.test.ts index 51c4b565..72baaec4 100644 --- a/test/services/extensions/v2/publisher.test.ts +++ b/test/services/extensions/v2/publisher.test.ts @@ -15,6 +15,7 @@ import { } from "./harness"; import { insertUser, + getDeveloper, insertExtension, bumpDeveloperOwnership } from "./db-fixtures"; @@ -54,7 +55,10 @@ async function registerPublisher(canPublish = true): Promise { return put( `/extensions/v2/m2m/clients/${CLIENT}`, await authHeaders(MODERATOR), - { developer_id: DEVELOPER, can_publish: canPublish } + { + developer_id: (await getDeveloper(db, DEVELOPER))!.id, + can_publish: canPublish + } ); } @@ -80,7 +84,12 @@ describe("m2m publisher registration", () => { const res = await put( `/extensions/v2/m2m/clients/${CLIENT}`, await authHeaders(OWNER), - { developer_id: DEVELOPER, can_publish: true } + { + developer_id: + (await getDeveloper(db, DEVELOPER))?.id ?? + "019a0000-0000-7000-8000-000000000001", + can_publish: true + } ); expect(res.status).toBe(403); }); @@ -100,7 +109,7 @@ describe("m2m publisher registration", () => { }; expect(body.result).toMatchObject({ client_id: CLIENT, - developer_id: DEVELOPER, + developer_id: (await getDeveloper(db, DEVELOPER))!.id, can_publish: true, revoked_at: null }); @@ -117,13 +126,19 @@ describe("m2m publisher registration", () => { const missing = await put( `/extensions/v2/m2m/clients/${CLIENT}`, await authHeaders(MODERATOR), - { developer_id: "no-such-developer", can_publish: true } + { + developer_id: "019a0000-0000-7000-8000-000000000001", + can_publish: true + } ); expect(missing.status).toBe(404); const unclaimed = await put( `/extensions/v2/m2m/clients/${CLIENT}`, await authHeaders(MODERATOR), - { developer_id: "unclaimed-dev", can_publish: true } + { + developer_id: (await getDeveloper(db, "unclaimed-dev"))!.id, + can_publish: true + } ); expect(unclaimed.status).toBe(404); }); @@ -168,9 +183,11 @@ describe("m2m publisher writes", () => { }; const revision = await revisionRow(body.result.revision_id); expect(revision?.submitted_by).toBe(`m2m:${CLIENT}`); - expect(revision?.developer_id).toBe(DEVELOPER); + expect(revision?.developer_id).toBe( + (await getDeveloper(db, DEVELOPER))!.id + ); const ext = await db - .prepare("SELECT created_by FROM extensions WHERE id = ?") + .prepare("SELECT created_by FROM extensions WHERE identifier= ?") .bind("published-ext") .first<{ created_by: string }>(); expect(ext?.created_by).toBe(OWNER); @@ -179,8 +196,8 @@ describe("m2m publisher writes", () => { it("proposes an edit to an extension of the mapped developer", async () => { await registerPublisher(); await insertExtension(db, { - id: "existing-ext", - developer_id: DEVELOPER, + identifier: "existing-ext", + developer_identifier: DEVELOPER, name: "Existing", source: '{"type":"github","repo":"example/existing"}' }); @@ -195,12 +212,21 @@ describe("m2m publisher writes", () => { }; const revision = await revisionRow(body.result.revision_id); expect(revision?.submitted_by).toBe(`m2m:${CLIENT}`); + const repeated = await put( + "/extensions/v2/extensions/existing-ext", + await m2mHeaders(["extensions:push"]), + sampleContent({ name: "Another Edit" }) + ); + expect(repeated.status).toBe(409); + await expect(repeated.json()).resolves.toMatchObject({ + error: { message: "An edit to this extension is already awaiting review" } + }); }); it("rejects writes for an unregistered client", async () => { await insertExtension(db, { - id: "existing-ext", - developer_id: DEVELOPER, + identifier: "existing-ext", + developer_identifier: DEVELOPER, name: "Existing", source: '{"type":"github","repo":"example/existing"}' }); @@ -338,7 +364,7 @@ describe("m2m publisher auto-approval", () => { ); const ext = await db .prepare( - "SELECT published_at, published_revision_id FROM extensions WHERE id = ?" + "SELECT published_at, published_revision_id FROM extensions WHERE identifier= ?" ) .bind("auto-published") .first<{ published_at: string | null; published_revision_id: string }>(); diff --git a/test/services/extensions/v2/resource-bounds.test.ts b/test/services/extensions/v2/resource-bounds.test.ts index 6d031ced..b8ab4732 100644 --- a/test/services/extensions/v2/resource-bounds.test.ts +++ b/test/services/extensions/v2/resource-bounds.test.ts @@ -1,3 +1,4 @@ +import { UUID_V7 } from "./db-fixtures"; import { describe, expect, it, vi } from "vitest"; import { createExecutionContext, @@ -39,6 +40,7 @@ import { } from "./harness"; import { insertExtension, + getDeveloper, insertRevision, insertUser, countExtensions, @@ -63,12 +65,15 @@ async function withdraw(id: string, user = "owner") { async function create(id: string) { return post("/extensions/v2/extensions", await authHeaders("owner"), { ...sampleCreate(), - id + identifier: id }); } async function owned() { await seedDeveloper("developer", "owner"); - await insertExtension(db, { id: "live", developer_id: "developer" }); + await insertExtension(db, { + identifier: "live", + developer_identifier: "developer" + }); } async function ageEvents(seconds = 120) { await db @@ -79,8 +84,8 @@ async function ageEvents(seconds = 120) { async function insertHistory(id: string, status = "rejected") { await insertRevision(db, { id, - extension_id: "live", - developer_id: "developer", + extension_identifier: "live", + developer_identifier: "developer", submitted_by: "owner", content: JSON.stringify(sampleContent()), status, @@ -130,11 +135,13 @@ describe("Extension resource admission", () => { it("keeps oversized published extensions visible as published cards", async () => { await owned(); await db - .prepare("UPDATE extensions SET readme=? WHERE id='live'") + .prepare("UPDATE extensions SET readme=? WHERE identifier='live'") .bind("x".repeat(MAX_CONTENT_BYTES + 1)) .run(); const extensions = new ExtensionsDatabase(getExtensionsDb(db)); - const list = await extensions.listOwned({ developerId: "developer" }); + const list = await extensions.listOwned({ + developerId: (await getDeveloper(db, "developer"))!.id + }); expect(list.error).toBeNull(); expect(list.data?.items[0].published).not.toBeNull(); expect( @@ -174,8 +181,8 @@ describe("Extension resource admission", () => { .first() ).toEqual({ submitted_by: "owner", - developer_id: "developer", - extension_id: "live", + developer_id: expect.stringMatching(UUID_V7), + extension_id: expect.stringMatching(UUID_V7), id: "fixed-attribution" }); }); @@ -256,7 +263,9 @@ describe("Extension resource admission", () => { ]; for (let i = 0; i < columns.length; i++) { await db - .prepare(`UPDATE extensions SET ${columns[i]}=? WHERE id='live'`) + .prepare( + `UPDATE extensions SET ${columns[i]}=? WHERE identifier='live'` + ) .bind(values[i]) .run(); await reconcile(); @@ -270,7 +279,7 @@ describe("Extension resource admission", () => { .bind(JSON.stringify({ ...content, readme: "😀" })) .run(); await reconcile(); - await db.prepare("DELETE FROM extensions WHERE id='live'").run(); + await db.prepare("DELETE FROM extensions WHERE identifier='live'").run(); await reconcile(); }); it("preserves overflow status when stream cancellation rejects", async () => { @@ -295,7 +304,7 @@ describe("Extension resource admission", () => { it("accepts a valid JSON body delivered in small chunks", async () => { await owned(); const bytes = new TextEncoder().encode( - JSON.stringify({ ...sampleCreate(), id: "chunked" }) + JSON.stringify({ ...sampleCreate(), identifier: "chunked" }) ); let position = 0; const stream = new ReadableStream({ @@ -320,7 +329,10 @@ describe("Extension resource admission", () => { await owned(); const id = "e".repeat(201); expect((await create(id)).status).toBe(422); - await insertExtension(db, { id, developer_id: "developer" }); + await insertExtension(db, { + identifier: id, + developer_identifier: "developer" + }); expect((await get(`/extensions/v2/extensions/${id}`, {})).status).toBe(200); expect( ( @@ -380,7 +392,7 @@ describe("Extension resource admission", () => { }); it("accepts the exact raw-byte cap and rejects one byte more without Content-Length", async () => { await seedDeveloper("developer", "owner"); - const body = JSON.stringify({ ...sampleCreate(), id: "exact" }); + const body = JSON.stringify({ ...sampleCreate(), identifier: "exact" }); const padded = body + " ".repeat( @@ -526,7 +538,9 @@ describe("Extension resource admission", () => { expect((await create(`churn-${i}`)).status).toBe(201); expect((await withdraw(`churn-${i}`)).status).toBe(200); } - await db.prepare("DELETE FROM developers WHERE id='developer'").run(); + await db + .prepare("DELETE FROM developers WHERE identifier = 'developer'") + .run(); await seedDeveloper("replacement", "owner"); await db .prepare("UPDATE users SET deleted_at=CURRENT_TIMESTAMP WHERE id='owner'") @@ -605,14 +619,25 @@ describe("Extension resource admission", () => { .prepare( `SELECT ${column} AS value FROM extension_resource_usage WHERE scope=? AND subject=?` ) - .bind(scope, subject) + .bind( + scope, + scope === "developer" + ? (await getDeveloper(db, subject))!.id + : subject + ) .first("value"); try { await db .prepare( `UPDATE extension_resource_usage SET ${column}=? WHERE scope=? AND subject=?` ) - .bind(limit, scope, subject) + .bind( + limit, + scope, + scope === "developer" + ? (await getDeveloper(db, subject))!.id + : subject + ) .run(); expect((await create("blocked")).status).toBe(409); expect(await countExtensions(db)).toBe(1); @@ -624,7 +649,13 @@ describe("Extension resource admission", () => { .prepare( `UPDATE extension_resource_usage SET ${column}=? WHERE scope=? AND subject=?` ) - .bind(original, scope, subject) + .bind( + original, + scope, + scope === "developer" + ? (await getDeveloper(db, subject))!.id + : subject + ) .run(); } } @@ -635,7 +666,7 @@ describe("Extension resource admission", () => { expect((await create("publication")).status).toBe(201); const revision = await db .prepare( - "SELECT id FROM extension_revisions WHERE extension_id='publication'" + "SELECT id FROM extension_revisions WHERE extension_id=(SELECT id FROM extensions WHERE identifier='publication')" ) .first("id"); await db @@ -661,14 +692,20 @@ describe("Extension resource admission", () => { ).toBe("pending"); expect( await db - .prepare("SELECT published_at FROM extensions WHERE id='publication'") + .prepare( + "SELECT published_at FROM extensions WHERE identifier='publication'" + ) .first("published_at") ).toBeNull(); }); it("accounts UTF-8 bytes exactly and releases them transactionally without shifting account charges", async () => { await seedDeveloper("developer", "owner"); - const content = { ...sampleCreate(), id: "multibyte", readme: "😀é" }; - const { id: _id, ...stored } = content; + const content = { + ...sampleCreate(), + identifier: "multibyte", + readme: "😀é" + }; + const { identifier: _identifier, ...stored } = content; expect( ( await post( @@ -689,7 +726,7 @@ describe("Extension resource admission", () => { await insertUser(db, { id: "recipient" }); await db .prepare( - "UPDATE developers SET owner_user_id='recipient' WHERE id='developer'" + "UPDATE developers SET owner_user_id='recipient' WHERE identifier = 'developer'" ) .run(); expect( @@ -774,8 +811,8 @@ describe("Bounded revision reads and maintenance", () => { ] as const) { await insertRevision(db, { id, - extension_id: "live", - developer_id: "developer", + extension_identifier: "live", + developer_identifier: "developer", submitted_by: "owner", content: JSON.stringify({ ...sampleContent(), releases }), status: "rejected", @@ -805,8 +842,8 @@ describe("Bounded revision reads and maintenance", () => { await owned(); await insertRevision(db, { id: "unicode-tag", - extension_id: "live", - developer_id: "developer", + extension_identifier: "live", + developer_identifier: "developer", submitted_by: "owner", content: JSON.stringify({ releases: [{ ...sampleContent().releases[0], tag: "😀".repeat(100) }] @@ -827,21 +864,21 @@ describe("Bounded revision reads and maintenance", () => { for (const id of ["old", "published"]) await insertHistory(id, "approved"); await db .prepare( - "UPDATE extensions SET published_revision_id='published' WHERE id='live'" + "UPDATE extensions SET published_revision_id='published' WHERE identifier='live'" ) .run(); await insertRevision(db, { id: "pending", - extension_id: "live", - developer_id: "developer", + extension_identifier: "live", + developer_identifier: "developer", submitted_by: "owner", content: JSON.stringify(sampleContent()), created_at: "2000-01-01 00:00:00" }); await insertRevision(db, { id: "recent", - extension_id: "live", - developer_id: "developer", + extension_identifier: "live", + developer_identifier: "developer", submitted_by: "owner", content: JSON.stringify(sampleContent()), status: "rejected", @@ -931,7 +968,10 @@ describe("Bounded revision reads and maintenance", () => { it("bounds response amplification for 100 maximum-sized bodies", async () => { await owned(); await seedDeveloper("second", "second-owner"); - await insertExtension(db, { id: "second-live", developer_id: "second" }); + await insertExtension(db, { + identifier: "second-live", + developer_identifier: "second" + }); await insertUser(db, { id: "mod", is_moderator: 1 }); const content = { ...sampleContent(), @@ -947,8 +987,8 @@ describe("Bounded revision reads and maintenance", () => { for (let i = 0; i < 100; i++) await insertRevision(db, { id: `max-${i}`, - extension_id: i < 50 ? "live" : "second-live", - developer_id: i < 50 ? "developer" : "second", + extension_identifier: i < 50 ? "live" : "second-live", + developer_identifier: i < 50 ? "developer" : "second", submitted_by: i < 50 ? "owner" : "second-owner", content: JSON.stringify(content), status: "rejected", @@ -1085,7 +1125,7 @@ describe("Bounded revision reads and maintenance", () => { async (source, type) => { await owned(); await db - .prepare("UPDATE extensions SET source=? WHERE id='live'") + .prepare("UPDATE extensions SET source=? WHERE identifier='live'") .bind(JSON.stringify(source)) .run(); const extensions = new ExtensionsDatabase(getExtensionsDb(db)); @@ -1095,7 +1135,7 @@ describe("Bounded revision reads and maintenance", () => { expect(publicCard?.source).toEqual({ type, repo: "example/repo" }); expect(ExtensionListItemSchema.safeParse(publicCard).success).toBe(true); const ownerCards = await extensions.listOwned({ - developerId: "developer" + developerId: (await getDeveloper(db, "developer"))!.id }); expect(ownerCards.error).toBeNull(); const ownerCard = ownerCards.data?.items[0]; @@ -1106,7 +1146,9 @@ describe("Bounded revision reads and maintenance", () => { expect(OwnedExtensionListItemSchema.safeParse(ownerCard).success).toBe( true ); - expect((await extensions.getById("live")).data?.source).toEqual(source); + expect((await extensions.getByIdentifier("live")).data?.source).toEqual( + source + ); } ); it("preserves under-limit legacy fields in public and owner details", async () => { @@ -1116,7 +1158,7 @@ describe("Bounded revision reads and maintenance", () => { const source = { type: "custom", repo: "R".repeat(5000) }; await db .prepare( - "UPDATE extensions SET website=?, license=?, source=? WHERE id='live'" + "UPDATE extensions SET website=?, license=?, source=? WHERE identifier='live'" ) .bind(website, JSON.stringify(license), JSON.stringify(source)) .run(); @@ -1130,7 +1172,7 @@ describe("Bounded revision reads and maintenance", () => { license: "L".repeat(1000), source: { type: "custom", repo: "R".repeat(500) } }); - const publicDetail = await extensions.getById("live"); + const publicDetail = await extensions.getByIdentifier("live"); expect(publicDetail.error).toBeNull(); expect(publicDetail.data).toMatchObject({ homepage_url: website, @@ -1151,12 +1193,12 @@ describe("Bounded revision reads and maintenance", () => { const download = "https://example.test/" + "y".repeat(3000); await db .prepare( - "UPDATE extensions SET website=?,download_url=?,releases=? WHERE id='live'" + "UPDATE extensions SET website=?,download_url=?,releases=? WHERE identifier='live'" ) .bind(homepage, download, JSON.stringify(sampleContent().releases)) .run(); const extensions = new ExtensionsDatabase(getExtensionsDb(db)); - const detail = await extensions.getById("live"); + const detail = await extensions.getByIdentifier("live"); expect(detail.error).toBeNull(); expect(ExtensionSchema.safeParse(detail.data).success).toBe(true); expect(detail.data).toMatchObject({ diff --git a/worker-configuration.d.ts b/worker-configuration.d.ts index bc5958a3..e43d65e3 100644 --- a/worker-configuration.d.ts +++ b/worker-configuration.d.ts @@ -1,5 +1,5 @@ /* eslint-disable */ -// Generated by Wrangler by running `wrangler types --env-interface=CloudflareBindings` (hash: 26fdf3e2c27ec137e614462c60dbc01c) +// Generated by Wrangler by running `wrangler types --env-interface=CloudflareBindings` (hash: e026844df6fb1a40db3d15ddf5939ab5) // Runtime types generated with workerd@1.20261103.1 2026-10-01 interface __BaseEnv_CloudflareBindings { AUTH_KV: KVNamespace; @@ -36,7 +36,6 @@ interface __BaseEnv_CloudflareBindings { EXTENSION_ASSETS_S3_ENDPOINT?: "https://s3.eu-central-003.backblazeb2.com"; EXTENSION_ASSETS_S3_REGION?: "eu-central-003"; EXTENSION_ASSETS_S3_BUCKET?: "fb-extensions-assets"; - EXTENSION_ASSETS_S3_PREFIX?: "production"; EXTENSION_ASSETS_PUBLIC_URL?: "https://extensions-assets.fossbilling.net"; } declare namespace Cloudflare { @@ -80,7 +79,7 @@ type StringifyValues> = { [Binding in keyof EnvType]: EnvType[Binding] extends string ? EnvType[Binding] : string; }; declare namespace NodeJS { - interface ProcessEnv extends StringifyValues> {} + interface ProcessEnv extends StringifyValues> {} } declare module "*.txt" { const value: string; diff --git a/wrangler.jsonc b/wrangler.jsonc index 553bca07..b22cbe42 100644 --- a/wrangler.jsonc +++ b/wrangler.jsonc @@ -33,7 +33,6 @@ "EXTENSION_ASSETS_S3_ENDPOINT": "https://s3.eu-central-003.backblazeb2.com", "EXTENSION_ASSETS_S3_REGION": "eu-central-003", "EXTENSION_ASSETS_S3_BUCKET": "fb-extensions-assets", - "EXTENSION_ASSETS_S3_PREFIX": "production", "EXTENSION_ASSETS_PUBLIC_URL": "https://extensions-assets.fossbilling.net", "EXTENSIONS_RETENTION_MODE": "dry-run", "EXTENSIONS_V2_EMAIL_PROVIDER": "mxroute", From ff6a54b999c752d2478ea64155b08d442c538a48 Mon Sep 17 00:00:00 2001 From: Adam Daley Date: Sat, 7 Nov 2026 20:01:42 +0000 Subject: [PATCH 04/12] Store prepared extension icons in B2 assets --- scripts/package-demo.ts | 2 +- src/services/extensions/v2/PACKAGE-RUNTIME.md | 15 +- src/services/extensions/v2/PACKAGING.md | 66 +++--- src/services/extensions/v2/db/extensions.ts | 61 +++-- .../extensions/v2/packages/b2-store.ts | 65 ++++-- src/services/extensions/v2/packages/build.ts | 37 ++- .../extensions/v2/packages/public-assets.ts | 184 +++------------ src/services/extensions/v2/packages/store.ts | 40 ++-- .../extensions/v2/routes/moderation.ts | 6 +- src/services/extensions/v2/routes/packages.ts | 3 - test/services/extensions/v2/b2-store.test.ts | 63 +++++- .../extensions/v2/package-builder.test.ts | 8 +- test/services/extensions/v2/packages.test.ts | 213 +++++++++++------- vitest.config.ts | 2 +- worker-configuration.d.ts | 4 +- wrangler.jsonc | 11 +- 16 files changed, 423 insertions(+), 357 deletions(-) diff --git a/scripts/package-demo.ts b/scripts/package-demo.ts index 77760716..d19b2c54 100644 --- a/scripts/package-demo.ts +++ b/scripts/package-demo.ts @@ -273,7 +273,7 @@ if (action === "setup") { unpublishedCorrection: next.id, sourceId: artifact.source_id, checks: - "Private queue status, private package and icon, approval, public checksum and icon, immutable correction" + "Private queue status, private package and API preview, approval, public checksum and icon, immutable correction" }, null, 2 diff --git a/src/services/extensions/v2/PACKAGE-RUNTIME.md b/src/services/extensions/v2/PACKAGE-RUNTIME.md index 106ccc70..9c3e7587 100644 --- a/src/services/extensions/v2/PACKAGE-RUNTIME.md +++ b/src/services/extensions/v2/PACKAGE-RUNTIME.md @@ -18,15 +18,24 @@ npm run packages:dev npm run packages:demo -- --fresh ``` -A fresh run verifies private job status, package and icon access before approval, -owner preview security headers, review and approval, the public ZIP checksum and -icon, and a private manifest correction using the stored source. +A fresh run verifies private job status, private package and API preview access +before approval, preview security headers, review and approval, the public ZIP +checksum and icon, and a private manifest correction using the stored source. Worker and Node tests additionally cover archive limits and corruption, queue retries and duplicate delivery, interrupted leases, ownership changes and publisher revocation. The directory tests cover the private job proxy and polling failures; publisher tests cover completion, rejection and the polling deadline. +## Real B2 smoke test + +A local Worker smoke test against both real B2 buckets on 7 November 2026 +verified pending SVG/AVIF/PNG CDN delivery, checksums, security headers and cache +hits. Approval and manifest corrections reused existing images. Cleanup removed +abandoned object versions while preserving referenced objects. Test objects and +CDN cache entries were removed afterward. No Worker deployment or production +database changes were made. + ## Before production enablement Measure admission and consumer invocations separately: CPU and wall duration, diff --git a/src/services/extensions/v2/PACKAGING.md b/src/services/extensions/v2/PACKAGING.md index 2e288e05..ca184827 100644 --- a/src/services/extensions/v2/PACKAGING.md +++ b/src/services/extensions/v2/PACKAGING.md @@ -3,7 +3,7 @@ Authors upload ready-to-install distribution ZIPs, with any compilation already complete. The API validates and repacks them with directory-generated manifests, then publishes them through the existing revision review. It never runs uploaded -code or package build scripts. Package storage uses the private B2 bucket when configured, otherwise Wrangler's local R2 emulator. Pending images continue to use the private image binding. +code or package build scripts. Package storage uses the private B2 bucket when configured, otherwise Wrangler's local R2 emulator. Prepared icons use the public asset bucket, including icons for pending submissions. Package routes are disabled by default. Local development and production enablement are separate modes. ## Run the local demo @@ -130,8 +130,10 @@ conflicts and corrupt archives are rejected. Checksums cover the final stored ZI Include at most one root `icon.svg`, `icon.png`, `icon.webp`, or `icon.avif`. There is no manifest icon field; packages without an icon use the generic icon. -Packaging prepares display images once in `EXTENSION_IMAGE_BUCKET`, shared by -all artifacts using the same source. The original icon stays unchanged in the ZIP. +Packaging prepares display images once and stores them in `fb-extensions-assets`. +The original icon stays unchanged in the ZIP. A new extension receives its permanent +UUIDv7 draft identity after package validation, before the first icon is stored. +The first review submission reuses that draft identity. SVGs are sanitized with Cloudflare svg-hush through the pinned `@zeropress/svg-hush` Workers adapter and served directly. SVG inputs are bounded by the ZIP expansion @@ -141,13 +143,19 @@ transparency without enlarging small images. Requests prefer SVG when present, otherwise AVIF when explicitly accepted, with PNG fallback. They read stored images directly and never reopen a ZIP. -Pending previews use the private image route with authorization, private cache -headers, `nosniff`, and a sandboxed Content Security Policy. They are never copied -to public B2. After approval, a background publication job copies the selected -icon to B2, then updates the published listing to its direct CDN URL. A failed -copy retains the existing image route; subsequent published-image reads retry. -The revision ID is checked before changing the listing URL, so a slow copy -cannot overwrite a newer publication. +All prepared icons, including pending ones, use +`extensions//icons/.` in the public asset bucket. +One asset UUID identifies all format variants; manifest corrections reuse them. +UUID paths are unlisted, not access control: anyone given the direct URL can read +the image before approval. ZIPs and package metadata remain private until approved. +The API preview route still checks ownership and returns private cache headers, +`nosniff`, and a sandboxed Content Security Policy. + +Icons are uploaded during packaging, with immutable public cache headers. An asset +upload failure fails the job; it never switches to another storage backend. +Approval updates the listing to the existing CDN URL and performs no image copy. +The revision ID is checked before changing the listing URL to avoid overwriting a +newer publication. ## Private B2 package testing @@ -168,7 +176,7 @@ EXTENSION_PACKAGES_S3_PREFIX="local" Keep this bucket private. Use a separate development key restricted to the bucket with read, write and delete-file permissions. No public URL or domain is required. Use `EXTENSION_PACKAGES_MODE=local` only outside production. The committed production configuration remains disabled. -With no S3 endpoint, ZIPs use the local R2 binding. An incomplete S3 configuration +Without S3 endpoints, ZIPs and prepared icons use the same local R2 emulator binding. An incomplete S3 configuration or upstream failure returns 503 rather than silently switching storage backends. R2 validates the supplied SHA-256 when writing and returns that checksum as @@ -179,14 +187,14 @@ Source and built ZIPs use `local/packages/`. Object IDs are allocated once by D1 before upload. Uploads carry a signed SHA-256 payload and private cache headers. Reads are bounded to 17 MiB and verify artifact bytes before returning a ZIP through the existing permission-checked download route. B2 URLs and credentials -are never returned to visitors. Images still use the separate private image -binding, with only approved display images copied to the public asset bucket. +are never returned to visitors. Images use the separate public asset bucket and +its CDN hostname, including before approval. Cleanup identifies the stored B2 version and deletes it explicitly. Deleting only by name would leave billable versions behind. Failed deletion retains the D1 reservation for retry. -## B2 and Cloudflare testing +## B2 asset and Cloudflare testing Add these to the local API's ignored `.dev.vars` (use `.dev.vars.packages-local` for the standalone API demo): @@ -200,7 +208,9 @@ EXTENSION_ASSETS_S3_SECRET_ACCESS_KEY="your-applicationKey" EXTENSION_ASSETS_PUBLIC_URL="https://extensions-assets.fossbilling.net" ``` -An unset endpoint disables B2 publication. Public object keys use +Restrict the asset key to this bucket with read, write and delete-file permissions. +Pending API previews read these stored objects; cleanup deletes abandoned versions. +An unset endpoint keeps assets in the local emulator instead of B2. Public object keys use `extensions//icons/.`, for example: ```text @@ -208,13 +218,13 @@ https://extensions-assets.fossbilling.net/extensions/019a0000-0000-7000-8000-000 ``` The parent is the extension record's UUID, not its public identifier. The icon -record uses its immutable package-source UUID as its asset identity. All format -variants share that identity, and manifest-only edits reuse the same image URL. +record allocates an immutable asset UUID when its display images are prepared. +All format variants share that identity, and manifest-only edits reuse the same image URL. A different source gets a new asset UUID. Uploads set `Cache-Control: public, max-age=31536000, immutable` and retain each file's SHA-256 in object metadata for verification. -The API checks existing objects before uploading and caches successful checks in -KV. Directory pages use direct CDN images with AVIF/PNG `` fallback, +Package source records retain the prepared image keys, so retries and metadata +corrections reuse existing images. Directory pages use direct CDN images with AVIF/PNG `` fallback, bypassing the directory image transformer. The bucket is public. The hostname-scoped Cloudflare response-header rule @@ -226,8 +236,8 @@ Content-Security-Policy: sandbox; default-src 'none'; style-src 'unsafe-inline'; X-Content-Type-Options: nosniff ``` -Only approved assets enter this bucket. Delisting stops access through the API, -but does not revoke previously published public image URLs or cached copies. +Prepared assets enter this bucket during packaging, before approval. Delisting +stops access through the API, but does not revoke direct image URLs or cached copies. Keep B2 daily caps and alerts configured for download and transaction charges. ## Production configuration (prepared locally) @@ -238,10 +248,9 @@ created by a local build or type-generation command. Before deployment (including an automatic deployment after merge): -1. Provision the `extension-packages` Queue and the private R2 bucket `fb-extensions-private-images` used by - `EXTENSION_IMAGE_BUCKET`. Do not attach a public domain or enable public R2 - access. This stores prepared pending icons; ZIPs remain in private B2. - The bindings must exist even while package mode is disabled; that flag blocks +1. Provision the `extension-packages` Queue. No R2 image bucket is needed; + ZIPs use private B2 and prepared icons use the public B2 asset bucket. + The queue binding must exist even while package mode is disabled; that flag blocks application routes, not deployment-time binding validation. 2. Supply `EXTENSION_PACKAGES_S3_ACCESS_KEY_ID` and `EXTENSION_PACKAGES_S3_SECRET_ACCESS_KEY` as Worker secrets for @@ -258,15 +267,14 @@ Before deployment (including an automatic deployment after merge): merging a disabled API alone does not make package submission available. Production requires HTTPS download URLs with the `/extensions/v2` base path, -complete private B2 credentials, an explicit non-`local` object prefix, private -image storage and the Images binding. An incomplete configuration fails with 503; +complete credentials for both B2 buckets, an explicit non-`local` ZIP object prefix, +a public asset URL and the Images binding. An incomplete configuration fails with 503; it cannot fall back to local ZIP storage. Private package objects use the `production` prefix in this configuration. Public image objects use the UUID-based layout described above. The disabled mode blocks package reads as well as writes. Once managed releases are published, disabling it also stops their API downloads and private previews; -it is not an upload-only pause switch. Public CDN images already published remain -accessible independently. +it is not an upload-only pause switch. Prepared CDN images remain accessible independently, even for pending packages. Local validation and the remaining production measurements are summarized in [PACKAGE-RUNTIME.md](PACKAGE-RUNTIME.md). Local queue success does not establish diff --git a/src/services/extensions/v2/db/extensions.ts b/src/services/extensions/v2/db/extensions.ts index 6cef99ff..a164300d 100644 --- a/src/services/extensions/v2/db/extensions.ts +++ b/src/services/extensions/v2/db/extensions.ts @@ -626,11 +626,10 @@ export class ExtensionsDatabase { } } - // Creates the extension record and its first pending revision as one - // transaction. The revision insert is gated on `changes() = 1` from the - // preceding statement (SQLite's per-connection changes()), so an id - // collision or a failed ownership guard leaves neither row behind. See - // toD1Statement for why this is raw sql rather than two builder calls. + // Creates the first pending revision, reusing a package draft when present. + // The batch gates the revision insert on changes() = 1, so a conflicting + // identifier or failed ownership guard cannot create a revision. A package + // draft is reused only when it has never been submitted. async create( input: CreateExtensionInput ): Promise< @@ -660,7 +659,11 @@ export class ExtensionsDatabase { SELECT COUNT(*) FROM extension_revisions WHERE submitted_by = ? AND status = 'pending' ) < ? - ON CONFLICT DO NOTHING`, + ON CONFLICT (lower(identifier)) DO UPDATE SET updated_at=CURRENT_TIMESTAMP + WHERE extensions.developer_id=excluded.developer_id + AND extensions.published_at IS NULL + AND NOT EXISTS (SELECT 1 FROM extension_revisions r WHERE r.extension_id=extensions.id) + RETURNING id`, params: [ extensionId, input.identifier, @@ -676,19 +679,22 @@ export class ExtensionsDatabase { const revisionStmt = toD1Statement(this.db.$client, { sql: `INSERT INTO extension_revisions (id, extension_id, developer_id, submitted_by, status, content, ownership_epoch) - SELECT ?, ?, ?, ?, 'pending', ?, ? - WHERE changes() = 1`, + SELECT ?, e.id, ?, ?, 'pending', ?, ? FROM extensions e + WHERE lower(e.identifier)=lower(?) AND changes() = 1`, params: [ revisionId, - extensionId, input.developerId, input.submittedBy, JSON.stringify(input.content), - input.ownershipEpoch + input.ownershipEpoch, + input.identifier ] }); - results = await this.db.$client.batch([extensionStmt, revisionStmt]); + results = await this.db.$client.batch<{ id: string }>([ + extensionStmt, + revisionStmt + ]); } catch (error) { return contentAdmissionError("create", error); } @@ -702,7 +708,11 @@ export class ExtensionsDatabase { } return { - data: { id: extensionId, identifier: input.identifier, revisionId }, + data: { + id: results[0].results[0].id, + identifier: input.identifier, + revisionId + }, error: null }; } @@ -771,7 +781,11 @@ export class ExtensionsDatabase { SELECT COUNT(*) FROM extension_revisions WHERE submitted_by = ? AND status = 'pending' ) < ? - ON CONFLICT DO NOTHING`, + ON CONFLICT (lower(identifier)) DO UPDATE SET updated_at=CURRENT_TIMESTAMP + WHERE extensions.developer_id=excluded.developer_id + AND extensions.published_at IS NULL + AND NOT EXISTS (SELECT 1 FROM extension_revisions r WHERE r.extension_id=extensions.id) + RETURNING id`, params: [ extensionId, input.identifier, @@ -785,20 +799,23 @@ export class ExtensionsDatabase { const revisionStmt = toD1Statement(this.db.$client, { sql: `INSERT INTO extension_revisions (id, extension_id, developer_id, submitted_by, status, content, ownership_epoch) - SELECT ?, ?, d.id, ?, 'pending', ?, d.ownership_epoch - FROM developers d - WHERE d.id = ? + SELECT ?, e.id, d.id, ?, 'pending', ?, d.ownership_epoch + FROM developers d JOIN extensions e ON e.developer_id=d.id + WHERE d.id = ? AND lower(e.identifier)=lower(?) AND changes() = 1`, params: [ revisionId, - extensionId, input.submittedBy, JSON.stringify(input.content), - input.developerId + input.developerId, + input.identifier ] }); - results = await this.db.$client.batch([extensionStmt, revisionStmt]); + results = await this.db.$client.batch<{ id: string }>([ + extensionStmt, + revisionStmt + ]); } catch (error) { return contentAdmissionError("createAsPublisher", error); } @@ -815,7 +832,11 @@ export class ExtensionsDatabase { } return { - data: { id: extensionId, identifier: input.identifier, revisionId }, + data: { + id: results[0].results[0].id, + identifier: input.identifier, + revisionId + }, error: null }; } diff --git a/src/services/extensions/v2/packages/b2-store.ts b/src/services/extensions/v2/packages/b2-store.ts index 503286ee..53a37e2d 100644 --- a/src/services/extensions/v2/packages/b2-store.ts +++ b/src/services/extensions/v2/packages/b2-store.ts @@ -6,18 +6,22 @@ import { boundedBytes, type PackageStore } from "./store"; // Includes generated manifest and ZIP headers as well as the source contents. const MAX_STORED_BYTES = PACKAGE_LIMITS.expandedBytes + 1024 * 1024; -export function b2PackageStore( - getEnv: (name: string) => string | undefined +export function b2ObjectStore( + getEnv: (name: string) => string | undefined, + kind: "packages" | "assets" ): PackageStore { const get = (suffix: string) => { - const value = getEnv(`EXTENSION_PACKAGES_S3_${suffix}`); + const value = getEnv(`EXTENSION_${kind.toUpperCase()}_S3_${suffix}`); if (!value) throw unavailable("Package storage configuration is incomplete"); return value; }; const origin = new URL(get("ENDPOINT")); const bucket = get("BUCKET"); - const prefix = getEnv("EXTENSION_PACKAGES_S3_PREFIX") ?? "local"; + const prefix = + kind === "packages" + ? (getEnv("EXTENSION_PACKAGES_S3_PREFIX") ?? "local") + : ""; if ( origin.protocol !== "https:" || origin.username || @@ -26,9 +30,25 @@ export function b2PackageStore( origin.hash || origin.pathname !== "/" || !/^[a-zA-Z0-9-]+$/.test(bucket) || - !/^[a-zA-Z0-9_-]+$/.test(prefix) + (kind === "packages" && !/^[a-zA-Z0-9_-]+$/.test(prefix)) ) throw unavailable("Invalid package storage configuration"); + if (kind === "assets") { + let base: URL; + try { + base = new URL(getEnv("EXTENSION_ASSETS_PUBLIC_URL") ?? ""); + } catch { + throw unavailable("Invalid public asset URL configuration"); + } + if ( + base.protocol !== "https:" || + base.username || + base.password || + base.search || + base.hash + ) + throw unavailable("Invalid public asset URL configuration"); + } const aws = new AwsClient({ accessKeyId: get("ACCESS_KEY_ID"), secretAccessKey: get("SECRET_ACCESS_KEY"), @@ -36,15 +56,18 @@ export function b2PackageStore( service: "s3", retries: 0 }); + const uuid = "[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}"; + const pattern = + kind === "assets" + ? new RegExp(`^extensions/${uuid}/icons/${uuid}\\.(?:svg|png|avif)$`) + : new RegExp(`^${uuid}$`); function target(key: string) { - // Object IDs are allocated once in D1 before a write, never chosen by callers. - if ( - !/^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/.test( - key - ) - ) - throw unavailable("Invalid package object ID"); - return new URL(`/${bucket}/${prefix}/packages/${key}`, origin); + // Keys come from D1 reservations, never from caller-supplied paths. + if (!pattern.test(key)) throw unavailable("Invalid storage object ID"); + return new URL( + `/${bucket}/${kind === "packages" ? `${prefix}/packages/` : ""}${key}`, + origin + ); } async function request(url: URL, init: RequestInit) { try { @@ -70,8 +93,18 @@ export function b2PackageStore( method: "PUT", body: bytes, headers: { - "Content-Type": "application/zip", - "Cache-Control": "private, no-store", + "Content-Type": + kind === "packages" + ? "application/zip" + : key.endsWith(".svg") + ? "image/svg+xml" + : key.endsWith(".avif") + ? "image/avif" + : "image/png", + "Cache-Control": + kind === "packages" + ? "private, no-store" + : "public, max-age=31536000, immutable", "x-amz-content-sha256": checksum, "x-amz-meta-sha256": checksum } @@ -94,7 +127,7 @@ export function b2PackageStore( "Stored package is unavailable or has a different checksum" ); } - // Verify the actual bytes before returning a downloadable ZIP. The size + // Verify actual bytes before returning an object. The size // bound also applies to responses without Content-Length (e.g. chunked). let bytes: Uint8Array; try { diff --git a/src/services/extensions/v2/packages/build.ts b/src/services/extensions/v2/packages/build.ts index 89190457..0e05171e 100644 --- a/src/services/extensions/v2/packages/build.ts +++ b/src/services/extensions/v2/packages/build.ts @@ -1,3 +1,4 @@ +import { v7 as uuidv7 } from "uuid"; import type { PublisherContext } from "../middleware"; import { sanitizeSvg } from "@zeropress/svg-hush/workerd"; import { @@ -75,11 +76,8 @@ export async function reserveObject( extensionId: string, bytes: Uint8Array, checksum: string, - suffix?: "avif" | "png" | "svg" + id: string = crypto.randomUUID() ) { - const id = suffix - ? `icons/${crypto.randomUUID()}.${suffix}` - : crypto.randomUUID(); const result = await db .prepare( `INSERT INTO package_objects (id,created_by,bytes) @@ -277,8 +275,35 @@ export async function buildStoredPackage(input: { ); } } + // Reserve the permanent identity only after ZIP and icon validation succeeds. + await db + .prepare( + `INSERT INTO extensions (id,identifier,developer_id,created_by) + SELECT ?,?,?,? WHERE ${ownerPredicate()} ON CONFLICT DO NOTHING` + ) + .bind( + uuidv7(), + manifest.id, + owner.id, + owner.owner_user_id, + ...ownerParams(owner, manifest.id) + ) + .run(); + const extension = await db + .prepare( + "SELECT id FROM extensions WHERE lower(identifier)=lower(?) AND developer_id=?" + ) + .bind(manifest.id, owner.id) + .first<{ id: string }>(); + if (!extension) + throw new PackageError( + "FORBIDDEN", + "Extension ownership changed while preparing the package", + 403 + ); await storePackageIcon({ db, + extensionId: extension.id, store, owner, manifest, @@ -334,6 +359,7 @@ export async function buildStoredPackage(input: { } async function storePackageIcon(input: { + extensionId: string; db: D1Database; store: PackageStore; owner: PackageOwner; @@ -349,6 +375,7 @@ async function storePackageIcon(input: { .first() ) return; + const assetId = crypto.randomUUID(); const objects: { format: IconVariant["format"]; id: string; @@ -365,7 +392,7 @@ async function storePackageIcon(input: { manifest.id, bytes, checksum, - format + `extensions/${input.extensionId}/icons/${assetId}.${format}` ), checksum }); diff --git a/src/services/extensions/v2/packages/public-assets.ts b/src/services/extensions/v2/packages/public-assets.ts index 9d315840..e23dcd43 100644 --- a/src/services/extensions/v2/packages/public-assets.ts +++ b/src/services/extensions/v2/packages/public-assets.ts @@ -1,190 +1,70 @@ import { logError } from "../../../../lib/logger"; import { revalidateExtensions } from "../revalidate"; -import type { ExtensionContent } from "../schemas/extensions"; -import { AwsClient } from "aws4fetch"; import type { Context } from "hono"; import { getPlatform } from "../../../../lib/middleware"; -import { ARTIFACT_SELECT, type ArtifactRow } from "../db/packages"; -import { latestRelease } from "../../../../lib/releases"; import { PackageError } from "./errors"; -import { boundedBytes, packageStore } from "./store"; -// Only called for committed, published content. Uploads and pending previews -// always use the private image bucket, even when B2 is configured. -export async function publishExtensionIcon( +// Publishing changes the listing URL only; icons are already stored in B2. +async function updatePublishedIconUrl( c: Context<{ Bindings: CloudflareBindings }>, extensionId: string ) { extensionId = extensionId.toLowerCase(); const platform = getPlatform(c); - const endpoint = platform.getEnv("EXTENSION_ASSETS_S3_ENDPOINT"); - if (!endpoint) return; - const get = (name: string) => { - const value = platform.getEnv(`EXTENSION_ASSETS_${name}`); - if (!value) - throw new PackageError( - "ASSETS_UNAVAILABLE", - "Asset storage configuration is incomplete", - 503 - ); - return value; - }; const row = await c.env.DB_EXTENSIONS.prepare( - "SELECT id,releases,published_revision_id FROM extensions WHERE lower(identifier)=? AND published_at IS NOT NULL AND delisted_at IS NULL" + ` + SELECT e.published_revision_id, coalesce(i.svg_object_id,i.png_object_id) AS object_key + FROM extensions e + JOIN package_artifacts a ON a.download_url=e.download_url AND a.published_at IS NOT NULL + JOIN package_sources s ON s.id=a.source_id AND s.extension_identifier=lower(e.identifier) + JOIN package_icons i ON i.source_id=s.id + WHERE lower(e.identifier)=? AND e.published_at IS NOT NULL AND e.delisted_at IS NULL + ` ) .bind(extensionId) - .first<{ id: string; releases: string; published_revision_id: string }>(); - if (!row) return; - const latest = latestRelease( - JSON.parse(row.releases) as ExtensionContent["releases"] - ); - if (!latest?.artifact_id) return; - const artifact = await c.env.DB_EXTENSIONS.prepare( - `${ARTIFACT_SELECT} WHERE a.id=? AND a.published_at IS NOT NULL AND s.extension_identifier=?` - ) - .bind(latest.artifact_id, extensionId) - .first(); - if ( - !artifact?.icon_source_id || - (!artifact.svg_object_id && !artifact.png_object_id) - ) - return; - const store = packageStore(c); - if (!store) return; - const origin = new URL(endpoint); - const publicBase = new URL(get("PUBLIC_URL")); + .first<{ published_revision_id: string; object_key: string | null }>(); + if (!row?.object_key) return; + const base = platform.getEnv("EXTENSION_ASSETS_PUBLIC_URL"); + if (!base) + throw new PackageError( + "ASSETS_UNAVAILABLE", + "Public asset URL is not configured", + 503 + ); + const publicBase = new URL(base); if ( - origin.protocol !== "https:" || publicBase.protocol !== "https:" || - origin.username || - origin.password || publicBase.username || publicBase.password || - origin.search || publicBase.search || - origin.hash || - publicBase.hash || - origin.pathname !== "/" + publicBase.hash ) throw new PackageError( "ASSETS_UNAVAILABLE", - "Asset storage URLs must use HTTPS without credentials or query parameters", - 503 - ); - const bucket = get("S3_BUCKET"); - if (!/^[a-zA-Z0-9-]+$/.test(bucket)) - throw new PackageError( - "ASSETS_UNAVAILABLE", - "Invalid asset bucket name", + "Invalid public asset URL", 503 ); - const aws = new AwsClient({ - accessKeyId: get("S3_ACCESS_KEY_ID"), - secretAccessKey: get("S3_SECRET_ACCESS_KEY"), - region: get("S3_REGION"), - service: "s3", - retries: 0 - }); - // The icon shares its immutable source UUID across formats and manifest revisions. - const assetId = artifact.icon_source_id; - let iconUrl: string | undefined; - for (const format of artifact.svg_object_id - ? (["svg"] as const) - : (["avif", "png"] as const)) { - const key = artifact[`${format}_object_id`]; - const checksum = artifact[`${format}_sha256`]; - if (!key || !checksum) - throw new PackageError( - "ASSETS_UNAVAILABLE", - "Prepared asset is unavailable", - 503 - ); - const objectKey = `extensions/${row.id}/icons/${assetId}.${format}`; - const url = new URL(`${publicBase.href.replace(/\/$/, "")}/${objectKey}`) - .href; - const contentType = format === "svg" ? "image/svg+xml" : `image/${format}`; - const cacheKey = `extension-asset:${url}`; - if (!(await c.env.CACHE_KV.get(cacheKey))) { - const target = new URL(`/${bucket}/${objectKey}`, origin); - const head = await fetch( - await aws.sign(target, { - method: "HEAD", - redirect: "manual", - signal: AbortSignal.timeout(5000) - }) - ); - if ( - head.status !== 404 && - (!head.ok || head.headers.get("x-amz-meta-sha256") !== checksum) - ) { - await head.body?.cancel(); - throw new PackageError( - "ASSETS_UNAVAILABLE", - "Unable to verify public asset storage", - 503 - ); - } - if (head.status === 404) { - const bytes = await boundedBytes( - await store.get(key, checksum), - 4 * 1024 * 1024 - ); - if (!bytes.length) - throw new PackageError( - "ASSETS_UNAVAILABLE", - "Prepared asset is unavailable", - 503 - ); - const response = await fetch( - await aws.sign(target, { - method: "PUT", - body: bytes, - redirect: "manual", - signal: AbortSignal.timeout(5000), - headers: { - "Content-Type": contentType, - "Cache-Control": "public, max-age=31536000, immutable", - "x-amz-content-sha256": checksum, - "x-amz-meta-sha256": checksum - } - }) - ); - await response.body?.cancel(); - if (!response.ok) - throw new PackageError( - "ASSETS_UNAVAILABLE", - "Public asset upload failed", - 503 - ); - } - await head.body?.cancel(); - await c.env.CACHE_KV.put(cacheKey, "ready", { expirationTtl: 86400 }); - } - // PNG is the fallback URL; the directory offers its AVIF sibling via . - if (format !== "avif") iconUrl = url; - } - if (iconUrl) { - const updated = await c.env.DB_EXTENSIONS.prepare( - "UPDATE extensions SET icon_url=? WHERE lower(identifier)=? AND published_revision_id=? AND delisted_at IS NULL AND icon_url IS NOT ?" - ) - .bind(iconUrl, extensionId, row.published_revision_id, iconUrl) - .run(); - return Boolean(updated.meta.changes); - } + const iconUrl = `${publicBase.href.replace(/\/$/, "")}/${row.object_key}`; + const updated = await c.env.DB_EXTENSIONS.prepare( + "UPDATE extensions SET icon_url=? WHERE lower(identifier)=? AND published_revision_id=? AND delisted_at IS NULL AND icon_url IS NOT ?" + ) + .bind(iconUrl, extensionId, row.published_revision_id, iconUrl) + .run(); + return Boolean(updated.meta.changes); } -export function scheduleIconPublication( +export function schedulePublishedIconUpdate( c: Context<{ Bindings: CloudflareBindings }>, extensionId: string ) { if (!getPlatform(c).getEnv("EXTENSION_ASSETS_S3_ENDPOINT")) return; c.executionCtx.waitUntil( - publishExtensionIcon(c, extensionId) + updatePublishedIconUrl(c, extensionId) .then((changed) => { if (changed) revalidateExtensions(c); }) .catch((error) => { - logError("extensions/packages", "Icon publication failed", { + logError("extensions/packages", "Published icon URL update failed", { extensionId, error: error instanceof Error ? error.message : String(error) }); diff --git a/src/services/extensions/v2/packages/store.ts b/src/services/extensions/v2/packages/store.ts index 4299f65e..9eb4e899 100644 --- a/src/services/extensions/v2/packages/store.ts +++ b/src/services/extensions/v2/packages/store.ts @@ -1,6 +1,6 @@ import { PackageError } from "./errors"; import type { Context } from "hono"; -import { b2PackageStore } from "./b2-store"; +import { b2ObjectStore } from "./b2-store"; import { getPlatform } from "../../../../lib/middleware"; export interface PackageStore { @@ -51,7 +51,8 @@ export function configuredPackageStore( !getEnv("EXTENSION_PACKAGES_S3_ENDPOINT") || !getEnv("EXTENSION_PACKAGES_S3_PREFIX") || getEnv("EXTENSION_PACKAGES_S3_PREFIX") === "local" || - !env.EXTENSION_IMAGE_BUCKET || + !getEnv("EXTENSION_ASSETS_S3_ENDPOINT") || + !getEnv("EXTENSION_ASSETS_PUBLIC_URL") || !env.PACKAGE_IMAGES ) { throw new PackageError( @@ -61,27 +62,30 @@ export function configuredPackageStore( ); } } - const remote = getEnv("EXTENSION_PACKAGES_S3_ENDPOINT") - ? b2PackageStore((name) => getEnv(name)) + const packages = getEnv("EXTENSION_PACKAGES_S3_ENDPOINT") + ? b2ObjectStore(getEnv, "packages") : null; - const packages = env.EXTENSION_PACKAGE_BUCKET; - const images = env.EXTENSION_IMAGE_BUCKET; - const bucketFor = (key: string) => { - const bucket = key.startsWith("icons/") ? images : packages; + const assets = getEnv("EXTENSION_ASSETS_S3_ENDPOINT") + ? b2ObjectStore(getEnv, "assets") + : null; + const bucket = env.EXTENSION_PACKAGE_BUCKET; + if (!packages && !bucket) return null; + const localBucket = () => { if (!bucket) throw new PackageError( "PACKAGES_DISABLED", - "Package or image storage is not configured", + "Storage is not configured", 503 ); return bucket; }; - if (!remote && !packages) return null; + const remoteFor = (key: string) => + key.startsWith("extensions/") ? assets : packages; return { async put(key, bytes, checksum) { - if (remote && !key.startsWith("icons/")) - return remote.put(key, bytes, checksum); - const result = await bucketFor(key).put(key, bytes, { + const remote = remoteFor(key); + if (remote) return remote.put(key, bytes, checksum); + const result = await localBucket().put(key, bytes, { onlyIf: { etagDoesNotMatch: "*" }, sha256: checksum, httpMetadata: { @@ -98,8 +102,9 @@ export function configuredPackageStore( if (!result) throw new Error("Package object already exists"); }, async get(key, checksum) { - if (remote && !key.startsWith("icons/")) return remote.get(key, checksum); - const object = await bucketFor(key).get(key); + const remote = remoteFor(key); + if (remote) return remote.get(key, checksum); + const object = await localBucket().get(key); if (!object) return null; if (checksum) { const actual = object.checksums.sha256; @@ -120,8 +125,9 @@ export function configuredPackageStore( return object.body; }, async delete(key) { - if (remote && !key.startsWith("icons/")) return remote.delete(key); - await bucketFor(key).delete(key); + const remote = remoteFor(key); + if (remote) return remote.delete(key); + await localBucket().delete(key); } }; } diff --git a/src/services/extensions/v2/routes/moderation.ts b/src/services/extensions/v2/routes/moderation.ts index 978e4c1d..5fb41f94 100644 --- a/src/services/extensions/v2/routes/moderation.ts +++ b/src/services/extensions/v2/routes/moderation.ts @@ -1,4 +1,4 @@ -import { scheduleIconPublication } from "../packages/public-assets"; +import { schedulePublishedIconUpdate } from "../packages/public-assets"; import { paceContentAccount, requireApproveAccess, @@ -119,7 +119,7 @@ export function registerModerationRoutes(app: ExtensionsV2App): void { const status = statusFromWriteErrorCode(error?.code); return c.json(errorBody(error, "Unable to approve revision"), status); } - scheduleIconPublication(c, id); + schedulePublishedIconUpdate(c, id); const notified = await notifyAuthor(c, extDb, query, { kind: "revision-approved", extensionId: id, @@ -418,7 +418,7 @@ export function registerModerationRoutes(app: ExtensionsV2App): void { const status = statusFromContentWriteError(error?.code); return c.json(errorBody(error, "Unable to correct extension"), status); } - scheduleIconPublication(c, id); + schedulePublishedIconUpdate(c, id); revalidateExtensions(c); return c.json( { diff --git a/src/services/extensions/v2/routes/packages.ts b/src/services/extensions/v2/routes/packages.ts index 0b97cca7..ba259a51 100644 --- a/src/services/extensions/v2/routes/packages.ts +++ b/src/services/extensions/v2/routes/packages.ts @@ -2,7 +2,6 @@ import { PackageJobsDatabase } from "../db/package-jobs"; import { dispatchPackageJob, type PackageJob } from "../packages/jobs"; import { canonicalJson } from "../db/packages"; import { sha256 } from "../package-builder"; -import { scheduleIconPublication } from "../packages/public-assets"; import { createRoute, z } from "@hono/zod-openapi"; import type { Context, MiddlewareHandler } from "hono"; import { getAuth } from "../../../../lib/auth"; @@ -446,8 +445,6 @@ export function registerPackageRoutes(app: ExtensionsV2App) { throw new PackageError("NOT_FOUND", "Package not found", 404); if (!download && !icon) return c.json(artifactResponse(row), 200); if (icon) { - if (row.published_at) - scheduleIconPublication(c, row.extension_identifier); const avif = (c.req.header("Accept") ?? "") .split(",") .some((part) => { diff --git a/test/services/extensions/v2/b2-store.test.ts b/test/services/extensions/v2/b2-store.test.ts index 56932294..f8f0f547 100644 --- a/test/services/extensions/v2/b2-store.test.ts +++ b/test/services/extensions/v2/b2-store.test.ts @@ -1,5 +1,5 @@ import { afterEach, describe, expect, it, vi } from "vitest"; -import { b2PackageStore } from "../../../../src/services/extensions/v2/packages/b2-store"; +import { b2ObjectStore } from "../../../../src/services/extensions/v2/packages/b2-store"; import { sha256, PACKAGE_LIMITS @@ -14,13 +14,14 @@ const config: Record = { SECRET_ACCESS_KEY: "test-secret" }; const store = (overrides: Record = {}) => - b2PackageStore( + b2ObjectStore( (name) => - ({ ...config, ...overrides })[name.replace("EXTENSION_PACKAGES_S3_", "")] + ({ ...config, ...overrides })[name.replace("EXTENSION_PACKAGES_S3_", "")], + "packages" ); afterEach(() => vi.unstubAllGlobals()); -describe("private B2 package storage", () => { +describe("B2 object storage", () => { it("signs ZIP writes and verifies downloaded bytes", async () => { const bytes = new Uint8Array(new TextEncoder().encode("synthetic ZIP")); const checksum = await sha256(bytes); @@ -56,6 +57,60 @@ describe("private B2 package storage", () => { expect(requests[0].headers.get("x-amz-content-sha256")).toBe(checksum); }); + it.each(["svg", "avif", "png"])( + "stores and cleans up public %s assets under their UUID path", + async (format) => { + const bytes = new Uint8Array(new TextEncoder().encode("prepared image")); + const checksum = await sha256(bytes); + const key = `extensions/${objectId}/icons/${objectId}.${format}`; + const requests: Request[] = []; + vi.stubGlobal("fetch", async (request: Request) => { + requests.push(request); + return new Response(request.method === "GET" ? bytes : null, { + headers: { + "x-amz-meta-sha256": checksum, + "x-amz-version-id": "asset-version" + } + }); + }); + const storage = b2ObjectStore( + (name) => + name === "EXTENSION_ASSETS_PUBLIC_URL" + ? "https://assets.example.test" + : { ...config, BUCKET: "public-assets" }[ + name.replace("EXTENSION_ASSETS_S3_", "") + ], + "assets" + ); + await storage.put(key, bytes, checksum); + expect(await new Response(await storage.get(key, checksum)).text()).toBe( + "prepared image" + ); + await storage.delete(key); + expect(requests.map((r) => r.method)).toEqual([ + "PUT", + "GET", + "HEAD", + "DELETE" + ]); + for (const request of requests) { + expect(new URL(request.url).pathname).toBe(`/public-assets/${key}`); + expect(request.headers.get("Authorization")).toContain( + "AWS4-HMAC-SHA256" + ); + } + expect(requests[0].headers.get("Content-Type")).toBe( + format === "svg" ? "image/svg+xml" : `image/${format}` + ); + expect(requests[0].headers.get("Cache-Control")).toBe( + "public, max-age=31536000, immutable" + ); + expect(new URL(requests[3].url).searchParams.get("versionId")).toBe( + "asset-version" + ); + } + ); + it("rejects corrupted bodies even when checksum metadata matches", async () => { const checksum = await sha256( new Uint8Array(new TextEncoder().encode("expected")) diff --git a/test/services/extensions/v2/package-builder.test.ts b/test/services/extensions/v2/package-builder.test.ts index 381ce041..8e8eef87 100644 --- a/test/services/extensions/v2/package-builder.test.ts +++ b/test/services/extensions/v2/package-builder.test.ts @@ -243,9 +243,11 @@ describe("local directory package builder", () => { const source = await zip([[filename, "unique-contents"]], { level: 0 }); - const offset = new TextDecoder() - .decode(source) - .indexOf("unique-contents"); + const contents = new TextEncoder().encode("unique-contents"); + // ZIP headers are binary: decoded string offsets are not byte offsets. + const offset = source.findIndex((_, index) => + contents.every((byte, part) => source[index + part] === byte) + ); expect(offset).toBeGreaterThan(0); source[offset] ^= 1; await expect(buildExtensionPackage(source, manifest)).rejects.toThrow(); diff --git a/test/services/extensions/v2/packages.test.ts b/test/services/extensions/v2/packages.test.ts index dce38b81..a9513bce 100644 --- a/test/services/extensions/v2/packages.test.ts +++ b/test/services/extensions/v2/packages.test.ts @@ -42,7 +42,7 @@ import { put, db } from "./harness"; -import { insertUser, getExtension, getDeveloper } from "./db-fixtures"; +import { insertUser, getExtension, getDeveloper, UUID_V7 } from "./db-fixtures"; vi.mock("@octokit/request", async () => (await import("../../../mocks/octokit")).octokitRequestMock() @@ -605,37 +605,15 @@ describe("queued package preparation", () => { }); describe("stored directory packages", () => { - it("uses private B2 ZIP storage through build, rebuild and approval without publishing pending images", async () => { - Object.assign(env, { + it("uses separate B2 buckets for ZIPs and icons through preparation, correction and approval", async () => { + const remote = configureB2({ EXTENSION_PACKAGES_S3_ENDPOINT: "https://s3.example.test", EXTENSION_PACKAGES_S3_REGION: "eu-central-003", EXTENSION_PACKAGES_S3_BUCKET: "test-packages", EXTENSION_PACKAGES_S3_ACCESS_KEY_ID: "test-key", EXTENSION_PACKAGES_S3_SECRET_ACCESS_KEY: "test-secret" }); - const objects = new Map(); - const requests: Request[] = []; - vi.stubGlobal("fetch", async (request: Request) => { - requests.push(request); - expect(request.headers.get("Authorization")).toMatch( - /^AWS4-HMAC-SHA256 / - ); - const key = new URL(request.url).pathname; - expect(key).toMatch(/^\/test-packages\/local\/packages\/[a-f0-9-]+$/); - if (request.method === "PUT") { - objects.set(key, { - bytes: await request.arrayBuffer(), - checksum: request.headers.get("x-amz-meta-sha256")! - }); - return new Response(null); - } - const stored = objects.get(key); - return stored - ? new Response(stored.bytes, { - headers: { "x-amz-meta-sha256": stored.checksum } - }) - : new Response(null, { status: 404 }); - }); + const requests = remote.calls; const writer = new ZipWriter(new Uint8ArrayWriter(), options); await writer.add("Service.php", new TextReader(" { first.source_id ); expect(revised.source_id).toBe(first.source_id); - expect(objects.size).toBe(3); + const uploads = requests.filter((request) => request.method === "PUT"); + expect(uploads).toHaveLength(4); // source, two manifests and one shared icon + expect( + uploads.filter((request) => + new URL(request.url).pathname.startsWith( + "/test-packages/local/packages/" + ) + ) + ).toHaveLength(3); + expect( + uploads.filter((request) => + new URL(request.url).pathname.startsWith("/test-assets/extensions/") + ) + ).toHaveLength(1); const path = `${root}/packages/${revised.id}/download`; const before = requests.length; expect((await get(path, {})).status).toBe(404); @@ -815,7 +806,9 @@ describe("stored directory packages", () => { avif_object_id: null; png_object_id: null; }>(); - expect(row?.svg_object_id).toMatch(/^icons\/.+\.svg$/); + expect(row?.svg_object_id).toMatch( + /^extensions\/[a-f0-9-]{36}\/icons\/[a-f0-9-]{36}\.svg$/ + ); expect(row?.avif_object_id).toBeNull(); expect(row?.png_object_id).toBeNull(); const download = await get( @@ -972,28 +965,40 @@ describe("stored directory packages", () => { expect((await upload()).status).toBe(201); }); - function configureB2() { + function configureB2(overrides: Record = {}) { Object.assign(env, { - EXTENSION_ASSETS_S3_ENDPOINT: "https://s3.eu-central-003.backblazeb2.com", + EXTENSION_ASSETS_S3_ENDPOINT: "https://s3.example.test", EXTENSION_ASSETS_S3_REGION: "eu-central-003", EXTENSION_ASSETS_S3_BUCKET: "test-assets", EXTENSION_ASSETS_S3_ACCESS_KEY_ID: "test-key", EXTENSION_ASSETS_S3_SECRET_ACCESS_KEY: "test-secret", - EXTENSION_ASSETS_PUBLIC_URL: "https://extensions-assets.fossbilling.net" + EXTENSION_ASSETS_PUBLIC_URL: "https://extensions-assets.fossbilling.net", + ...overrides }); const calls: Request[] = []; + const objects = new Map(); let failure = false; vi.stubGlobal( "fetch", async (input: RequestInfo | URL, init?: RequestInit) => { const request = input instanceof Request ? input : new Request(input, init); - expect(new URL(request.url).hostname).toBe( - "s3.eu-central-003.backblazeb2.com" - ); + expect(new URL(request.url).hostname).toBe("s3.example.test"); calls.push(request.clone()); - return new Response(null, { - status: failure ? 403 : request.method === "HEAD" ? 404 : 200 + if (failure) return new Response(null, { status: 503 }); + const key = new URL(request.url).pathname; + if (request.method === "PUT") { + objects.set(key, { + bytes: await request.arrayBuffer(), + checksum: request.headers.get("x-amz-meta-sha256")! + }); + return new Response(null); + } + const object = objects.get(key); + if (!object) return new Response(null, { status: 404 }); + expect(request.method).toBe("GET"); + return new Response(object.bytes, { + headers: { "x-amz-meta-sha256": object.checksum } }); } ); @@ -1026,43 +1031,54 @@ describe("stored directory packages", () => { ); return writer.close(); } - it("keeps pending images private and publishes a stable UUID icon URL", async () => { + it("stores pending icons at unlisted B2 URLs using the permanent draft UUID", async () => { const remote = configureB2(); const artifact = await built(manifest(), await iconSource()); - const pending = await submitted(artifact); + const draft = (await getExtension(db, "package-demo"))!; + expect(draft.id).toMatch(UUID_V7); + expect(draft.published_at).toBeNull(); + expect( + (await get(`${root}/extensions/package-demo`, await authHeaders("owner"))) + .status + ).toBe(200); + expect((await get(`${root}/extensions/package-demo`, {})).status).toBe(404); + const uploads = remote.calls.filter((r) => r.method === "PUT"); + expect(uploads).toHaveLength(1); + const upload = uploads[0]; + const objectKey = new URL(upload.url).pathname.replace("/test-assets/", ""); + expect(objectKey).toMatch( + new RegExp(`^extensions/${draft.id}/icons/[a-f0-9-]{36}\\.svg$`) + ); + const callsBefore = remote.calls.length; + expect((await get(`${root}/packages/${artifact.id}/icon`, {})).status).toBe( + 404 + ); expect( ( await get( `${root}/packages/${artifact.id}/icon`, - await authHeaders("owner") + await authHeaders("stranger") ) ).status - ).toBe(200); - expect(remote.calls).toHaveLength(0); + ).toBe(404); + expect(remote.calls).toHaveLength(callsBefore); + const preview = await get( + `${root}/packages/${artifact.id}/icon`, + await authHeaders("owner") + ); + expect(preview.status).toBe(200); + expect(preview.headers.get("Cache-Control")).toBe("private, no-store"); + expect(await preview.text()).toContain(" request.method)).toEqual([ - "HEAD", - "PUT" - ]); - const upload = remote.calls[1]; - expect(new URL(upload.url).pathname).toBe(`/test-assets/${objectKey}`); - expect(upload.headers.get("Authorization")).toContain( - "AWS4-HMAC-SHA256 Credential=test-key/" - ); - expect(upload.headers.get("Cache-Control")).toBe( - "public, max-age=31536000, immutable" - ); - expect(upload.headers.get("Content-Type")).toBe("image/svg+xml"); - expect(await sha256(new Uint8Array(await upload.arrayBuffer()))).toBe( - upload.headers.get("x-amz-meta-sha256") - ); - await get(`${root}/packages/${artifact.id}/icon`, {}); - expect(remote.calls).toHaveLength(2); + expect(remote.calls).toHaveLength(callsBeforeApproval); const edited = await built( { ...manifest(), description: "Updated metadata" }, undefined, @@ -1079,39 +1095,37 @@ describe("stored directory packages", () => { }; expect((await approve(result.revision_id)).status).toBe(200); expect((await getExtension(db, "package-demo"))?.icon_url).toBe( - published!.icon_url + published.icon_url ); - expect(remote.calls).toHaveLength(2); + expect(remote.calls.filter((r) => r.method === "PUT")).toHaveLength(1); }); - it("keeps approval and fallback delivery working during B2 failures, then retries publication", async () => { + it("fails preparation on asset upload errors and retries without changing storage backends", async () => { const remote = configureB2(); + const source = await iconSource(); remote.fail(true); - const artifact = await built(manifest(), await iconSource()); - expect((await approve(await submitted(artifact))).status).toBe(200); - expect((await getExtension(db, "package-demo"))?.icon_url).toBe( - artifact.download_url.replace(/\/download$/, "/icon") - ); - expect((await get(`${root}/packages/${artifact.id}/icon`, {})).status).toBe( - 200 - ); - expect(remote.calls.every((request) => request.method === "HEAD")).toBe( - true - ); + expect((await upload(manifest(), source)).status).toBe(503); + expect( + await db + .prepare("SELECT count(*) AS count FROM package_artifacts") + .first() + ).toEqual({ count: 0 }); remote.fail(false); - expect((await get(`${root}/packages/${artifact.id}/icon`, {})).status).toBe( - 200 - ); + const artifact = await built(manifest(), source); + expect( + ( + await get( + `${root}/packages/${artifact.id}/icon`, + await authHeaders("owner") + ) + ).status + ).toBe(200); + expect((await approve(await submitted(artifact))).status).toBe(200); expect((await getExtension(db, "package-demo"))?.icon_url).toContain( "https://extensions-assets.fossbilling.net/extensions/" ); }); it("publishes an icon when a valid extension ID is supplied in uppercase", async () => { configureB2(); - const fetch = vi.fn( - async (request: Request) => - new Response(null, { status: request.method === "HEAD" ? 404 : 200 }) - ); - vi.stubGlobal("fetch", fetch); const artifact = await built(manifest(), await iconSource()); const revision = await submitted(artifact); await insertUser(db, { id: "moderator", is_moderator: 1 }); @@ -1135,8 +1149,10 @@ describe("stored directory packages", () => { const uploads = remote.calls.filter((request) => request.method === "PUT"); expect(uploads).toHaveLength(2); const extension = (await getExtension(db, "package-demo"))!; - expect(new URL(uploads[0].url).pathname).toBe( - `/test-assets/extensions/${extension.id}/icons/${artifact.source_id}.avif` + expect(new URL(uploads[0].url).pathname).toMatch( + new RegExp( + `^/test-assets/extensions/${extension.id}/icons/[a-f0-9-]{36}\\.avif$` + ) ); expect(uploads[0].url.replace(/\.avif$/, ".png")).toBe(uploads[1].url); expect( @@ -1144,7 +1160,7 @@ describe("stored directory packages", () => { ).toEqual(["image/avif", "image/png"]); expect((await getExtension(db, "package-demo"))?.icon_url).toBe( uploads[1].url.replace( - "https://s3.eu-central-003.backblazeb2.com/test-assets", + "https://s3.example.test/test-assets", "https://extensions-assets.fossbilling.net" ) ); @@ -1214,6 +1230,16 @@ describe("stored directory packages", () => { await seedDeveloper("other-developer", "other"); expect( (await upload(manifest(), undefined, first.source_id, "other")).status + ).toBe(403); // The package has already reserved the extension for its owner. + expect( + ( + await upload( + { ...manifest(), id: "other-extension" }, + undefined, + first.source_id, + "other" + ) + ).status ).toBe(404); }); @@ -1248,6 +1274,11 @@ describe("stored directory packages", () => { "SELECT count(*) AS count FROM extensions WHERE identifier='package-demo'" ) .first() + ).toEqual({ count: 1 }); + expect( + await db + .prepare("SELECT count(*) AS count FROM extension_revisions") + .first() ).toEqual({ count: 0 }); }); @@ -1531,6 +1562,14 @@ describe("stored directory packages", () => { EXTENSION_PACKAGES_S3_ACCESS_KEY_ID: "test-key", EXTENSION_PACKAGES_S3_SECRET_ACCESS_KEY: "test-secret" }); + Object.assign(env, { + EXTENSION_ASSETS_S3_ENDPOINT: "https://s3.example.test", + EXTENSION_ASSETS_S3_REGION: "eu-central-003", + EXTENSION_ASSETS_S3_BUCKET: "public-assets", + EXTENSION_ASSETS_S3_ACCESS_KEY_ID: "test-key", + EXTENSION_ASSETS_S3_SECRET_ACCESS_KEY: "test-secret", + EXTENSION_ASSETS_PUBLIC_URL: "https://extensions-assets.fossbilling.net" + }); const requests: Request[] = []; vi.stubGlobal("fetch", async (request: Request) => { requests.push(request); diff --git a/vitest.config.ts b/vitest.config.ts index 6e9e4b8e..bdf5057d 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -18,7 +18,7 @@ export default defineConfig({ cloudflareTest({ wrangler: { configPath: "./wrangler.jsonc" }, miniflare: { - r2Buckets: ["EXTENSION_PACKAGE_BUCKET", "EXTENSION_IMAGE_BUCKET"], + r2Buckets: ["EXTENSION_PACKAGE_BUCKET"], images: { binding: "PACKAGE_IMAGES" }, // Add test environment variables bindings: { diff --git a/worker-configuration.d.ts b/worker-configuration.d.ts index e43d65e3..2d9ed9da 100644 --- a/worker-configuration.d.ts +++ b/worker-configuration.d.ts @@ -1,12 +1,11 @@ /* eslint-disable */ -// Generated by Wrangler by running `wrangler types --env-interface=CloudflareBindings` (hash: e026844df6fb1a40db3d15ddf5939ab5) +// Generated by Wrangler by running `wrangler types --env-interface=CloudflareBindings` (hash: 9b9b2dbaff5d17401395576e365b39a4) // Runtime types generated with workerd@1.20261103.1 2026-10-01 interface __BaseEnv_CloudflareBindings { AUTH_KV: KVNamespace; CACHE_KV: KVNamespace; DOWNLOAD_BUCKET: R2Bucket; EXTENSION_PACKAGE_BUCKET?: R2Bucket; - EXTENSION_IMAGE_BUCKET: R2Bucket; DB_CENTRAL_ALERTS: D1Database; DB_EXTENSIONS: D1Database; EXTENSION_PACKAGE_QUEUE: Queue; @@ -48,7 +47,6 @@ declare namespace Cloudflare { CACHE_KV: KVNamespace; DOWNLOAD_BUCKET: R2Bucket; EXTENSION_PACKAGE_BUCKET: R2Bucket; - EXTENSION_IMAGE_BUCKET: R2Bucket; DB_CENTRAL_ALERTS: D1Database; DB_EXTENSIONS: D1Database; EXTENSION_PACKAGE_QUEUE: Queue; diff --git a/wrangler.jsonc b/wrangler.jsonc index b22cbe42..f8c5f1b2 100644 --- a/wrangler.jsonc +++ b/wrangler.jsonc @@ -23,7 +23,7 @@ "keep_vars": true, "vars": { "ENVIRONMENT": "production", - // Enable only after the private image bucket and B2 secrets are provisioned. + // Enable only after the queue, Images binding and B2 secrets are configured. "EXTENSION_PACKAGES_MODE": "disabled", "EXTENSION_PACKAGES_BASE_URL": "https://api.fossbilling.net/extensions/v2", "EXTENSION_PACKAGES_S3_ENDPOINT": "https://s3.eu-central-003.backblazeb2.com", @@ -90,11 +90,6 @@ { "binding": "DOWNLOAD_BUCKET", "bucket_name": "fossbilling-download" - }, - { - // Private prepared icons, including pending submissions. Provision before deployment. - "binding": "EXTENSION_IMAGE_BUCKET", - "bucket_name": "fb-extensions-private-images" } ], "services": [ @@ -187,10 +182,6 @@ { "binding": "EXTENSION_PACKAGE_BUCKET", "bucket_name": "extension-packages-local" - }, - { - "binding": "EXTENSION_IMAGE_BUCKET", - "bucket_name": "extension-images-local" } ], "durable_objects": { From 38de33308e98c70240e725c54fef6c9abdfb8e61 Mon Sep 17 00:00:00 2001 From: Adam Daley Date: Mon, 9 Nov 2026 06:51:48 +0000 Subject: [PATCH 05/12] Clarify v2 version response differences --- src/services/extensions/v1/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/services/extensions/v1/README.md b/src/services/extensions/v1/README.md index c48b14cc..092f3da5 100644 --- a/src/services/extensions/v1/README.md +++ b/src/services/extensions/v1/README.md @@ -17,6 +17,8 @@ FOSSBilling 0.8.9 still uses v1. Upgrade integrations to [extensions v2](../v2/R | `GET /{id}` | `GET /extensions/v2/extensions/{id}` | | `GET /{id}/version` | Read `version` from the v2 detail response (the selected latest release, or newest stable release by default) | +The v2 version replacement returns JSON rather than plain text. Its `version` is stored without a leading `v`, while the v1 version endpoint preserves the release tag. For oversized published content, the v2 detail endpoint returns `409 CONTENT_UNAVAILABLE` instead of a version; handle that error before reading `result.version`. + V2 lists use cursor pagination, not v1 limit/offset pagination. Use type `module` instead of `mod`. Releases carry full `requires.fossbilling` constraints; consumers must evaluate that constraint rather than a derived minimum. ## Temporary badge rewrites From 30c1049d955bc7d1a5b79d634f0282ea0eb4fac3 Mon Sep 17 00:00:00 2001 From: Adam Daley Date: Mon, 9 Nov 2026 07:13:59 +0000 Subject: [PATCH 06/12] Tighten package manifest handling --- src/services/extensions/v2/db/revisions.ts | 3 +-- src/services/extensions/v2/packages/cloudflare-queue.ts | 4 ++-- src/services/extensions/v2/routes/packages.ts | 5 ++--- test/services/extensions/v2/packages.test.ts | 8 ++------ 4 files changed, 7 insertions(+), 13 deletions(-) diff --git a/src/services/extensions/v2/db/revisions.ts b/src/services/extensions/v2/db/revisions.ts index 98be2bf9..5799328d 100644 --- a/src/services/extensions/v2/db/revisions.ts +++ b/src/services/extensions/v2/db/revisions.ts @@ -1,5 +1,4 @@ -import { packageIconUrl } from "./packages"; -import { validatePackageContent } from "./packages"; +import { packageIconUrl, validatePackageContent } from "./packages"; import { manifestMetadata } from "../schemas/manifest"; import { MAX_CONTENT_BYTES } from "../resource-limits"; import { and, asc, desc, eq, gt, lt, or, sql, SQL } from "drizzle-orm"; diff --git a/src/services/extensions/v2/packages/cloudflare-queue.ts b/src/services/extensions/v2/packages/cloudflare-queue.ts index f7c93711..fb502a95 100644 --- a/src/services/extensions/v2/packages/cloudflare-queue.ts +++ b/src/services/extensions/v2/packages/cloudflare-queue.ts @@ -12,8 +12,8 @@ export async function prepareQueuedPackage( env: CloudflareBindings, job: PackageJob ): Promise { - const getEnv = (name: string) => - createCloudflareBindings(env).environment.get(name); + const bindings = createCloudflareBindings(env); + const getEnv = (name: string) => bindings.environment.get(name); const store = configuredPackageStore(env, getEnv); if (!store) throw new PackageError( diff --git a/src/services/extensions/v2/routes/packages.ts b/src/services/extensions/v2/routes/packages.ts index ba259a51..cbc0aed2 100644 --- a/src/services/extensions/v2/routes/packages.ts +++ b/src/services/extensions/v2/routes/packages.ts @@ -1,12 +1,11 @@ import { PackageJobsDatabase } from "../db/package-jobs"; import { dispatchPackageJob, type PackageJob } from "../packages/jobs"; -import { canonicalJson } from "../db/packages"; import { sha256 } from "../package-builder"; import { createRoute, z } from "@hono/zod-openapi"; import type { Context, MiddlewareHandler } from "hono"; import { getAuth } from "../../../../lib/auth"; import { getPlatform } from "../../../../lib/middleware"; -import { ARTIFACT_SELECT, ArtifactRow } from "../db/packages"; +import { ARTIFACT_SELECT, ArtifactRow, canonicalJson } from "../db/packages"; import { requireContentWrite, getPublisher, @@ -56,7 +55,7 @@ function artifactResponse(row: ArtifactRow) { result: { id: row.id, source_id: row.source_id, - manifest: ManifestSchema.loose().parse(JSON.parse(row.manifest)), + manifest: ManifestSchema.parse(JSON.parse(row.manifest)), sha256: row.sha256, download_url: row.download_url, published_at: row.published_at, diff --git a/test/services/extensions/v2/packages.test.ts b/test/services/extensions/v2/packages.test.ts index a9513bce..7ad4f4a1 100644 --- a/test/services/extensions/v2/packages.test.ts +++ b/test/services/extensions/v2/packages.test.ts @@ -80,9 +80,7 @@ afterEach(() => { "S3_SECRET_ACCESS_KEY", "PUBLIC_URL" ]) - delete (env as unknown as Record)[ - `EXTENSION_ASSETS_${key}` - ]; + Reflect.deleteProperty(env, `EXTENSION_ASSETS_${key}`); for (const suffix of [ "ENDPOINT", "REGION", @@ -91,9 +89,7 @@ afterEach(() => { "SECRET_ACCESS_KEY", "PREFIX" ]) - delete (env as unknown as Record)[ - `EXTENSION_PACKAGES_S3_${suffix}` - ]; + Reflect.deleteProperty(env, `EXTENSION_PACKAGES_S3_${suffix}`); env.EXTENSION_PACKAGES_MODE = "disabled"; }); async function zip(content = " Date: Mon, 9 Nov 2026 07:58:23 +0000 Subject: [PATCH 07/12] Isolate package runtime adapters and remove the demo --- package-lock.json | 1 - package.json | 6 +- scripts/package-demo.ts | 284 ------------- src/app/index.ts | 2 +- .../adapters/b2/package-store.ts} | 12 +- .../adapters/cloudflare/package-assets.ts} | 34 +- src/lib/adapters/cloudflare/package-images.ts | 62 +++ src/lib/adapters/cloudflare/package-jobs.ts | 88 ++++ src/lib/adapters/cloudflare/package-store.ts | 121 ++++++ src/services/extensions/v2/PACKAGE-RUNTIME.md | 25 +- src/services/extensions/v2/PACKAGING.md | 61 ++- .../extensions/v2/db/package-builds.ts | 330 +++++++++++++++ src/services/extensions/v2/db/packages.ts | 52 +-- src/services/extensions/v2/packages/build.ts | 396 +++--------------- .../v2/packages/cloudflare-queue.ts | 125 ------ src/services/extensions/v2/packages/images.ts | 10 + src/services/extensions/v2/packages/jobs.ts | 2 +- .../extensions/v2/packages/manifest.ts | 29 ++ .../extensions/v2/packages/prepare.ts | 70 ++++ .../extensions/v2/packages/repository.ts | 95 +++++ src/services/extensions/v2/packages/store.ts | 119 ------ .../extensions/v2/routes/moderation.ts | 2 +- src/services/extensions/v2/routes/packages.ts | 65 +-- test/env.d.ts | 1 + test/integration/scheduled.test.ts | 2 +- ...b2-store.test.ts => b2-store.node.test.ts} | 2 +- .../extensions/v2/package-build.node.test.ts | 174 ++++++++ test/services/extensions/v2/packages.test.ts | 40 +- vitest.node.config.ts | 2 +- worker-configuration.d.ts | 61 +-- wrangler.jsonc | 83 +--- 31 files changed, 1167 insertions(+), 1189 deletions(-) delete mode 100644 scripts/package-demo.ts rename src/{services/extensions/v2/packages/b2-store.ts => lib/adapters/b2/package-store.ts} (95%) rename src/{services/extensions/v2/packages/public-assets.ts => lib/adapters/cloudflare/package-assets.ts} (56%) create mode 100644 src/lib/adapters/cloudflare/package-images.ts create mode 100644 src/lib/adapters/cloudflare/package-jobs.ts create mode 100644 src/lib/adapters/cloudflare/package-store.ts create mode 100644 src/services/extensions/v2/db/package-builds.ts delete mode 100644 src/services/extensions/v2/packages/cloudflare-queue.ts create mode 100644 src/services/extensions/v2/packages/images.ts create mode 100644 src/services/extensions/v2/packages/manifest.ts create mode 100644 src/services/extensions/v2/packages/prepare.ts create mode 100644 src/services/extensions/v2/packages/repository.ts rename test/services/extensions/v2/{b2-store.test.ts => b2-store.node.test.ts} (98%) create mode 100644 test/services/extensions/v2/package-build.node.test.ts diff --git a/package-lock.json b/package-lock.json index 83c72a89..5b7d1725 100644 --- a/package-lock.json +++ b/package-lock.json @@ -35,7 +35,6 @@ "eslint-config-prettier": "10.1.8", "jiti": "2.7.0", "prettier": "3.9.10", - "tsx": "4.23.15", "typescript": "6.0.3", "typescript-eslint": "8.72.1", "vitest": "5.0.5", diff --git a/package.json b/package.json index 1718f1c3..d34ff7b2 100644 --- a/package.json +++ b/package.json @@ -22,10 +22,7 @@ "test:node": "vitest run --config vitest.node.config.ts", "test:all": "npm run test && npm run test:node", "test:coverage": "vitest run --coverage", - "typecheck": "tsc --noEmit", - "packages:setup": "tsx scripts/package-demo.ts setup", - "packages:dev": "wrangler dev --env packages-local --local --ip 127.0.0.1 --port 8789 --persist-to .wrangler/package-demo", - "packages:demo": "tsx scripts/package-demo.ts exercise" + "typecheck": "tsc --noEmit" }, "dependencies": { "@hono/zod-openapi": "1.6.3", @@ -56,7 +53,6 @@ "eslint-config-prettier": "10.1.8", "jiti": "2.7.0", "prettier": "3.9.10", - "tsx": "4.23.15", "typescript": "6.0.3", "typescript-eslint": "8.72.1", "vitest": "5.0.5", diff --git a/scripts/package-demo.ts b/scripts/package-demo.ts deleted file mode 100644 index d19b2c54..00000000 --- a/scripts/package-demo.ts +++ /dev/null @@ -1,284 +0,0 @@ -import { v7 as uuidv7 } from "uuid"; -import assert from "node:assert/strict"; -import { execFileSync } from "node:child_process"; -import { readFile, writeFile } from "node:fs/promises"; -import { sign } from "hono/jwt"; -import { TextReader, Uint8ArrayWriter, ZipWriter } from "@zip.js/zip.js"; -import { PackageArtifactSchema } from "../src/services/extensions/v2/schemas/packages"; -import { sha256 } from "../src/services/extensions/v2/package-builder"; - -const secretsFile = ".dev.vars.packages-local"; -const state = ".wrangler/package-demo"; -const base = "http://localhost:8789/extensions/v2"; -const action = process.argv[2]; -const fresh = process.argv.includes("--fresh"); -if (action === "setup") { - try { - await writeFile( - secretsFile, - `ASSERTION_SIGNING_SECRET="${Array.from(crypto.getRandomValues(new Uint8Array(32)), (b) => b.toString(16).padStart(2, "0")).join("")}"\nGITHUB_TOKEN=""\nEXTENSIONS_REVALIDATE_SECRET=""\n`, - { flag: "wx", mode: 0o600 } - ); - } catch (error) { - if (!(error instanceof Error && "code" in error && error.code === "EEXIST")) - throw error; - } - function wrangler(args: string[]) { - execFileSync( - process.execPath, - [ - "node_modules/wrangler/bin/wrangler.js", - ...args, - "--env", - "packages-local", - "--local", - "--persist-to", - state - ], - { - stdio: ["pipe", "pipe", "inherit"], - maxBuffer: 16 * 1024 * 1024, - env: { ...process.env, CI: "true" } - } - ); - } - for (const binding of ["DB_EXTENSIONS", "DB_CENTRAL_ALERTS"]) - wrangler(["d1", "migrations", "apply", binding]); - wrangler([ - "d1", - "execute", - "DB_EXTENSIONS", - "--command", - ` - INSERT INTO users (id,name,created_at,updated_at,is_moderator) VALUES - ('package-demo-owner','Local Package Developer',CURRENT_TIMESTAMP,CURRENT_TIMESTAMP,0), - ('package-demo-reviewer','Local Package Reviewer',CURRENT_TIMESTAMP,CURRENT_TIMESTAMP,1) - ON CONFLICT DO NOTHING; - INSERT INTO developers (id,identifier,type,name,owner_user_id,approved_at,created_at,updated_at) - VALUES ('${uuidv7()}','package-demo-studio','user','Local Package Developer','package-demo-owner',CURRENT_TIMESTAMP,CURRENT_TIMESTAMP,CURRENT_TIMESTAMP) - ON CONFLICT DO NOTHING; - ` - ]); - console.info( - "Local package demo initialized. Start npm run packages:dev, then run npm run packages:demo." - ); -} else if (action === "exercise") { - const secret = (await readFile(secretsFile, "utf8")).match( - /^ASSERTION_SIGNING_SECRET="([a-f0-9]+)"$/m - )?.[1]; - if (!secret) throw new Error("Run npm run packages:setup first"); - async function request( - path: string, - init: RequestInit = {}, - user?: string - ): Promise { - const headers = new Headers(init.headers); - if (user) { - const now = Math.floor(Date.now() / 1000); - headers.set( - "Authorization", - `Bearer ${await sign({ sub: user, iat: now, exp: now + 60, iss: "fossbilling-extensions", aud: "fossbilling-api/extensions-v2", purpose: "user-authentication", ver: 1 }, secret!, "HS256")}` - ); - } - const response = await fetch(`${base}${path}`, { ...init, headers }); - if (path !== "/packages" || response.status !== 202) return response; - assert.ok(response.headers.get("Location")?.includes("/package-jobs/")); - let job = ( - (await response.json()) as { - result: { - id: string; - status: string; - artifact: unknown; - error: { status: number; message: string } | null; - }; - } - ).result; - assert.equal((await request(`/package-jobs/${job.id}`)).status, 401); - for (let attempt = 0; attempt < 120; attempt++) { - if (job.status === "ready") - return Response.json({ result: job.artifact }, { status: 201 }); - if (job.status === "failed") - return Response.json( - { error: job.error }, - { status: job.error?.status ?? 422 } - ); - await new Promise((resolve) => setTimeout(resolve, 1000)); - const status = await request(`/package-jobs/${job.id}`, {}, user); - assert.equal(status.status, 200, await status.clone().text()); - job = ((await status.json()) as { result: typeof job }).result; - } - throw new Error(`Package job ${job.id} did not complete locally`); - } - const manifest = { - manifest_version: 1, - type: "module", - id: fresh - ? `package-demo-${crypto.randomUUID().slice(0, 8)}` - : "directory-package-demo", - name: "Directory Package Demo", - version: "0.1.0", - description: "A locally built extension package", - author: { name: "Local Package Developer" }, - license: "MIT", - requires: { fossbilling: ">=0.8.0" } - }; - const writer = new ZipWriter(new Uint8ArrayWriter(), { - useWebWorkers: false, - useCompressionStream: true, - lastModDate: new Date("1980-01-01T00:00:00Z"), - extendedTimestamp: false - }); - await writer.add( - "Service.php", - new TextReader("' - ) - ); - const source = await writer.close(); - const form = new FormData(); - form.set("manifest", JSON.stringify(manifest)); - form.set("source", new File([source], "source.zip")); - const uploaded = await request( - "/packages", - { method: "POST", body: form }, - "package-demo-owner" - ); - const body = (await uploaded.json()) as { result: unknown }; - assert.equal(uploaded.status, 201, JSON.stringify(body)); - const artifact = PackageArtifactSchema.parse(body.result); - const privateCheck = await request(`/packages/${artifact.id}/download`); - if (fresh) assert.equal(privateCheck.status, 404); - else assert.ok([404, 200].includes(privateCheck.status)); // 200 on a repeat. - await privateCheck.body?.cancel(); - const iconPath = `/packages/${artifact.id}/icon`; - if (fresh) assert.equal((await request(iconPath)).status, 404); - const preview = await request(iconPath, {}, "package-demo-owner"); - assert.equal(preview.status, 200); - assert.equal(preview.headers.get("Content-Type"), "image/svg+xml"); - assert.match(preview.headers.get("Cache-Control") ?? "", /private/); - assert.equal(preview.headers.get("X-Content-Type-Options"), "nosniff"); - assert.match(preview.headers.get("Content-Security-Policy") ?? "", /sandbox/); - await preview.body?.cancel(); - const { id: _id, ...metadata } = artifact.manifest; - const content = { - ...metadata, - readme: "# Directory Package Demo\n\nBuilt and reviewed locally.", - source: { - type: "custom", - repo: "https://example.invalid/local-package-demo" - }, - download_url: artifact.download_url, - releases: [ - { - tag: manifest.version, - date: "2026-11-05", - requires: manifest.requires, - artifact_id: artifact.id, - sha256: artifact.sha256, - download_url: artifact.download_url - } - ] - }; - const created = await request( - "/extensions", - { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ identifier: manifest.id, ...content }) - }, - "package-demo-owner" - ); - let revisionId: string | undefined; - if (created.status === 201) - revisionId = ((await created.json()) as { result: { revision_id: string } }) - .result.revision_id; - else { - assert.equal(created.status, 409, await created.text()); - const existing = await request( - `/extensions/${manifest.id}`, - {}, - "package-demo-owner" - ); - const listing = (await existing.json()) as { - result: { - pending_revision?: { id: string; content: { download_url: string } }; - published?: { download_url: string }; - }; - }; - if ( - listing.result.pending_revision?.content.download_url === - artifact.download_url - ) - revisionId = listing.result.pending_revision.id; - else - assert.equal( - listing.result.published?.download_url, - artifact.download_url, - "Existing demo has unrelated changes; it was left untouched" - ); - } - if (revisionId) { - const approved = await request( - `/extensions/${manifest.id}/revisions/${revisionId}/approve`, - { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: "{}" - }, - "package-demo-reviewer" - ); - assert.equal(approved.status, 200, await approved.text()); - } - const download = await request(`/packages/${artifact.id}/download`); - assert.equal(download.status, 200); - assert.equal( - await sha256(new Uint8Array(await download.arrayBuffer())), - artifact.sha256 - ); - const publicIcon = await request(iconPath); - assert.equal(publicIcon.status, 200); - assert.equal(publicIcon.headers.get("Content-Type"), "image/svg+xml"); - await publicIcon.body?.cancel(); - const correction = new FormData(); - correction.set( - "manifest", - JSON.stringify({ - ...manifest, - description: "A corrected manifest using the same source ZIP" - }) - ); - correction.set("source_id", artifact.source_id); - const corrected = await request( - "/packages", - { method: "POST", body: correction }, - "package-demo-owner" - ); - assert.equal(corrected.status, 201); - const next = PackageArtifactSchema.parse( - ((await corrected.json()) as { result: unknown }).result - ); - assert.equal(next.source_id, artifact.source_id); - assert.notEqual(next.sha256, artifact.sha256); - assert.equal((await request(`/packages/${next.id}/download`)).status, 404); - console.info( - JSON.stringify( - { - publishedPackage: artifact.download_url, - sha256: artifact.sha256, - unpublishedCorrection: next.id, - sourceId: artifact.source_id, - checks: - "Private queue status, private package and API preview, approval, public checksum and icon, immutable correction" - }, - null, - 2 - ) - ); -} else { - throw new Error("Use npm run packages:setup or npm run packages:demo"); -} diff --git a/src/app/index.ts b/src/app/index.ts index 4d769277..0f775643 100644 --- a/src/app/index.ts +++ b/src/app/index.ts @@ -1,7 +1,7 @@ import { consumePackageJobs, recoverPackageJobs -} from "../services/extensions/v2/packages/cloudflare-queue"; +} from "../lib/adapters/cloudflare/package-jobs"; import { getExtensionsDb } from "../lib/db"; import { maintainExtensionResources, diff --git a/src/services/extensions/v2/packages/b2-store.ts b/src/lib/adapters/b2/package-store.ts similarity index 95% rename from src/services/extensions/v2/packages/b2-store.ts rename to src/lib/adapters/b2/package-store.ts index 53a37e2d..09bf1465 100644 --- a/src/services/extensions/v2/packages/b2-store.ts +++ b/src/lib/adapters/b2/package-store.ts @@ -1,7 +1,13 @@ import { AwsClient } from "aws4fetch"; -import { PACKAGE_LIMITS, sha256 } from "../package-builder"; -import { PackageError } from "./errors"; -import { boundedBytes, type PackageStore } from "./store"; +import { + PACKAGE_LIMITS, + sha256 +} from "../../../services/extensions/v2/package-builder"; +import { PackageError } from "../../../services/extensions/v2/packages/errors"; +import { + boundedBytes, + type PackageStore +} from "../../../services/extensions/v2/packages/store"; // Includes generated manifest and ZIP headers as well as the source contents. const MAX_STORED_BYTES = PACKAGE_LIMITS.expandedBytes + 1024 * 1024; diff --git a/src/services/extensions/v2/packages/public-assets.ts b/src/lib/adapters/cloudflare/package-assets.ts similarity index 56% rename from src/services/extensions/v2/packages/public-assets.ts rename to src/lib/adapters/cloudflare/package-assets.ts index e23dcd43..82b2e283 100644 --- a/src/services/extensions/v2/packages/public-assets.ts +++ b/src/lib/adapters/cloudflare/package-assets.ts @@ -1,8 +1,9 @@ -import { logError } from "../../../../lib/logger"; -import { revalidateExtensions } from "../revalidate"; +import { PackageBuildDatabase } from "../../../services/extensions/v2/db/package-builds"; +import { logError } from "../../logger"; +import { revalidateExtensions } from "../../../services/extensions/v2/revalidate"; import type { Context } from "hono"; -import { getPlatform } from "../../../../lib/middleware"; -import { PackageError } from "./errors"; +import { getPlatform } from "../../middleware"; +import { PackageError } from "../../../services/extensions/v2/packages/errors"; // Publishing changes the listing URL only; icons are already stored in B2. async function updatePublishedIconUrl( @@ -11,18 +12,8 @@ async function updatePublishedIconUrl( ) { extensionId = extensionId.toLowerCase(); const platform = getPlatform(c); - const row = await c.env.DB_EXTENSIONS.prepare( - ` - SELECT e.published_revision_id, coalesce(i.svg_object_id,i.png_object_id) AS object_key - FROM extensions e - JOIN package_artifacts a ON a.download_url=e.download_url AND a.published_at IS NOT NULL - JOIN package_sources s ON s.id=a.source_id AND s.extension_identifier=lower(e.identifier) - JOIN package_icons i ON i.source_id=s.id - WHERE lower(e.identifier)=? AND e.published_at IS NOT NULL AND e.delisted_at IS NULL - ` - ) - .bind(extensionId) - .first<{ published_revision_id: string; object_key: string | null }>(); + const repository = new PackageBuildDatabase(c.env.DB_EXTENSIONS); + const row = await repository.publishedIcon(extensionId); if (!row?.object_key) return; const base = platform.getEnv("EXTENSION_ASSETS_PUBLIC_URL"); if (!base) @@ -45,12 +36,11 @@ async function updatePublishedIconUrl( 503 ); const iconUrl = `${publicBase.href.replace(/\/$/, "")}/${row.object_key}`; - const updated = await c.env.DB_EXTENSIONS.prepare( - "UPDATE extensions SET icon_url=? WHERE lower(identifier)=? AND published_revision_id=? AND delisted_at IS NULL AND icon_url IS NOT ?" - ) - .bind(iconUrl, extensionId, row.published_revision_id, iconUrl) - .run(); - return Boolean(updated.meta.changes); + return repository.updatePublishedIcon( + extensionId, + row.published_revision_id, + iconUrl + ); } export function schedulePublishedIconUpdate( diff --git a/src/lib/adapters/cloudflare/package-images.ts b/src/lib/adapters/cloudflare/package-images.ts new file mode 100644 index 00000000..9b08c0ea --- /dev/null +++ b/src/lib/adapters/cloudflare/package-images.ts @@ -0,0 +1,62 @@ +import { sanitizeSvg } from "@zeropress/svg-hush/workerd"; +import type { + PackageImageProcessor, + IconVariant +} from "../../../services/extensions/v2/packages/images"; +import { PackageError } from "../../../services/extensions/v2/packages/errors"; +import { boundedBytes } from "../../../services/extensions/v2/packages/store"; + +export class CloudflarePackageImages implements PackageImageProcessor { + constructor(private readonly images?: ImagesBinding) {} + async prepare( + icon: Uint8Array, + isSvg: boolean + ): Promise { + const images = this.images; + if (isSvg) { + try { + return [{ format: "svg", bytes: new Uint8Array(sanitizeSvg(icon)) }]; + } catch { + throw new PackageError( + "INVALID_ICON", + "Package SVG icon could not be sanitized", + 422 + ); + } + } + if (!images) + throw new PackageError( + "PACKAGES_DISABLED", + "Image processing is not configured", + 503 + ); + const variants: IconVariant[] = []; + try { + const info = await images.info(new Blob([icon]).stream()); + // Check decoded format too: a misleading filename must not bypass sanitization. + if (info.format === "image/svg+xml") + return [{ format: "svg", bytes: new Uint8Array(sanitizeSvg(icon)) }]; + for (const format of ["avif", "png"] as const) { + const output = await images + .input(new Blob([icon]).stream()) + .transform({ + width: 256, + height: 256, + fit: "scale-down" + }) + .output({ format: `image/${format}`, anim: false }); + variants.push({ + format, + bytes: await boundedBytes(output.image(), 1024 * 1024) + }); + } + } catch { + throw new PackageError( + "INVALID_ICON", + "Package icon could not be processed", + 422 + ); + } + return variants; + } +} diff --git a/src/lib/adapters/cloudflare/package-jobs.ts b/src/lib/adapters/cloudflare/package-jobs.ts new file mode 100644 index 00000000..70a1cee4 --- /dev/null +++ b/src/lib/adapters/cloudflare/package-jobs.ts @@ -0,0 +1,88 @@ +import { logError } from "../../logger"; +import { createCloudflareBindings } from "./index"; +import { PackageJobsDatabase } from "../../../services/extensions/v2/db/package-jobs"; +import { PackageBuildDatabase } from "../../../services/extensions/v2/db/package-builds"; +import { + prepareQueuedPackage, + type PackagePreparation +} from "../../../services/extensions/v2/packages/prepare"; +import { + processPackageJob, + dispatchPackageJob +} from "../../../services/extensions/v2/packages/jobs"; +import { PackageError } from "../../../services/extensions/v2/packages/errors"; +import { configuredPackageStore } from "./package-store"; +import { CloudflarePackageImages } from "./package-images"; + +type PackageBindings = CloudflareBindings & { + EXTENSION_PACKAGE_BUCKET?: R2Bucket; +}; +export function packagePreparation(env: PackageBindings): PackagePreparation { + const bindings = createCloudflareBindings(env); + const getEnv = (name: string) => bindings.environment.get(name); + const store = configuredPackageStore(env, getEnv); + const baseUrl = getEnv("EXTENSION_PACKAGES_BASE_URL"); + if (!store || !baseUrl) + throw new PackageError( + "PACKAGES_DISABLED", + "Package storage or download URL is not configured", + 503 + ); + return { + repository: new PackageBuildDatabase(env.DB_EXTENSIONS), + store, + images: new CloudflarePackageImages(env.PACKAGE_IMAGES), + baseUrl + }; +} + +export async function consumePackageJobs( + batch: MessageBatch<{ jobId: string }>, + env: CloudflareBindings +) { + const jobs = new PackageJobsDatabase(env.DB_EXTENSIONS); + for (const message of batch.messages) { + if ( + !message.body || + typeof message.body.jobId !== "string" || + !/^[a-f0-9-]{36}$/.test(message.body.jobId) + ) { + message.ack(); + continue; + } + try { + const result = await processPackageJob(jobs, message.body.jobId, (job) => + prepareQueuedPackage(packagePreparation(env), job) + ); + if (result === "busy") message.retry({ delaySeconds: 1200 }); + else if (result === "retry") message.retry({ delaySeconds: 60 }); + else message.ack(); + } catch { + message.retry({ delaySeconds: 60 }); + } + } +} + +export async function recoverPackageJobs(env: CloudflareBindings) { + const mode = createCloudflareBindings(env).environment.get( + "EXTENSION_PACKAGES_MODE" + ); + if ( + !env.EXTENSION_PACKAGE_QUEUE || + !["local", "enabled"].includes(mode ?? "") + ) + return; + const jobs = new PackageJobsDatabase(env.DB_EXTENSIONS); + await jobs.expire(); + const queue = createCloudflareBindings(env).queues!.EXTENSION_PACKAGE_QUEUE; + for (const id of await jobs.pending()) { + try { + await dispatchPackageJob(jobs, queue, id); + } catch (error) { + logError("extensions/packages", "Package job dispatch failed", { + jobId: id, + error: error instanceof Error ? error.message : String(error) + }); + } + } +} diff --git a/src/lib/adapters/cloudflare/package-store.ts b/src/lib/adapters/cloudflare/package-store.ts new file mode 100644 index 00000000..5bcf5aa7 --- /dev/null +++ b/src/lib/adapters/cloudflare/package-store.ts @@ -0,0 +1,121 @@ +import { PackageError } from "../../../services/extensions/v2/packages/errors"; +import type { PackageStore } from "../../../services/extensions/v2/packages/store"; +import type { Context } from "hono"; +import { b2ObjectStore } from "../b2/package-store"; +import { getPlatform } from "../../middleware"; + +export function packageStore( + c: Context<{ Bindings: CloudflareBindings }> +): PackageStore | null { + return configuredPackageStore(c.env, (name) => getPlatform(c).getEnv(name)); +} + +export function configuredPackageStore( + env: CloudflareBindings & { EXTENSION_PACKAGE_BUCKET?: R2Bucket }, + getEnv: (name: string) => string | undefined +): PackageStore | null { + const mode = getEnv("EXTENSION_PACKAGES_MODE"); + const production = getEnv("ENVIRONMENT") === "production"; + if (mode !== "enabled" && !(mode === "local" && !production)) return null; + if (production) { + const base = getEnv("EXTENSION_PACKAGES_BASE_URL"); + let url: URL; + try { + url = new URL(base ?? ""); + } catch { + throw new PackageError( + "PACKAGES_DISABLED", + "Package download URL is not configured", + 503 + ); + } + if ( + url.protocol !== "https:" || + url.username || + url.password || + url.search || + url.hash || + url.pathname !== "/extensions/v2" || + !getEnv("EXTENSION_PACKAGES_S3_ENDPOINT") || + !getEnv("EXTENSION_PACKAGES_S3_PREFIX") || + getEnv("EXTENSION_PACKAGES_S3_PREFIX") === "local" || + !getEnv("EXTENSION_ASSETS_S3_ENDPOINT") || + !getEnv("EXTENSION_ASSETS_PUBLIC_URL") || + !env.PACKAGE_IMAGES + ) { + throw new PackageError( + "PACKAGES_DISABLED", + "Production package storage, images or download URL is not configured", + 503 + ); + } + } + const packages = getEnv("EXTENSION_PACKAGES_S3_ENDPOINT") + ? b2ObjectStore(getEnv, "packages") + : null; + const assets = getEnv("EXTENSION_ASSETS_S3_ENDPOINT") + ? b2ObjectStore(getEnv, "assets") + : null; + const bucket = env.EXTENSION_PACKAGE_BUCKET; + if (!packages && !bucket) return null; + const localBucket = () => { + if (!bucket) + throw new PackageError( + "PACKAGES_DISABLED", + "Storage is not configured", + 503 + ); + return bucket; + }; + const remoteFor = (key: string) => + key.startsWith("extensions/") ? assets : packages; + return { + async put(key, bytes, checksum) { + const remote = remoteFor(key); + if (remote) return remote.put(key, bytes, checksum); + const result = await localBucket().put(key, bytes, { + onlyIf: { etagDoesNotMatch: "*" }, + sha256: checksum, + httpMetadata: { + contentType: key.endsWith(".avif") + ? "image/avif" + : key.endsWith(".png") + ? "image/png" + : key.endsWith(".svg") + ? "image/svg+xml" + : "application/zip", + cacheControl: "private, no-store" + } + }); + if (!result) throw new Error("Package object already exists"); + }, + async get(key, checksum) { + const remote = remoteFor(key); + if (remote) return remote.get(key, checksum); + const object = await localBucket().get(key); + if (!object) return null; + if (checksum) { + const actual = object.checksums.sha256; + const hex = actual + ? Array.from(new Uint8Array(actual), (byte) => + byte.toString(16).padStart(2, "0") + ).join("") + : null; + if (hex !== checksum) { + await object.body.cancel(); + throw new PackageError( + "PACKAGE_UNAVAILABLE", + "Stored object failed checksum verification", + 503 + ); + } + } + return object.body; + }, + async delete(key) { + const remote = remoteFor(key); + if (remote) return remote.delete(key); + await localBucket().delete(key); + } + }; +} diff --git a/src/services/extensions/v2/PACKAGE-RUNTIME.md b/src/services/extensions/v2/PACKAGE-RUNTIME.md index 9c3e7587..3d13bd59 100644 --- a/src/services/extensions/v2/PACKAGE-RUNTIME.md +++ b/src/services/extensions/v2/PACKAGE-RUNTIME.md @@ -9,23 +9,14 @@ compressed payloads, replaces the manifest and prepares the display icon. ## Local checks -The standalone demo uses the real Wrangler queue consumer with local D1 and R2: - -```sh -npm run packages:setup -npm run packages:dev -# In a second terminal: -npm run packages:demo -- --fresh -``` - -A fresh run verifies private job status, private package and API preview access -before approval, preview security headers, review and approval, the public ZIP -checksum and icon, and a private manifest correction using the stored source. - -Worker and Node tests additionally cover archive limits and corruption, queue -retries and duplicate delivery, interrupted leases, ownership changes and -publisher revocation. The directory tests cover the private job proxy and polling -failures; publisher tests cover completion, rejection and the polling deadline. +Run `npm run test:all`. Worker tests cover the API routes, guarded D1 writes, +archive limits and corruption, ownership changes, publisher revocation, private +previews, approval and cleanup. Storage and image bindings are emulated for these +tests. Node tests exercise package preparation and job processing through their +interfaces, plus the B2 adapter's request and integrity checks. + +The directory tests cover the private job proxy and polling failures; publisher +tests cover completion, rejection and the polling deadline. ## Real B2 smoke test diff --git a/src/services/extensions/v2/PACKAGING.md b/src/services/extensions/v2/PACKAGING.md index ca184827..02c90238 100644 --- a/src/services/extensions/v2/PACKAGING.md +++ b/src/services/extensions/v2/PACKAGING.md @@ -6,30 +6,18 @@ then publishes them through the existing revision review. It never runs uploaded code or package build scripts. Package storage uses the private B2 bucket when configured, otherwise Wrangler's local R2 emulator. Prepared icons use the public asset bucket, including icons for pending submissions. Package routes are disabled by default. Local development and production enablement are separate modes. -## Run the local demo +## Local development and testing -Use Node.js 24, install dependencies with `npm ci`, then: +Run the API through the normal `npm run dev` command. Use ignored `.dev.vars` +for local credentials, set `ENVIRONMENT=local` and `EXTENSION_PACKAGES_MODE=local`, +and set `EXTENSION_PACKAGES_BASE_URL` to the local API's `/extensions/v2` URL. +Apply local migrations with `npm run db:migrate:all:local`. The configured Queue +and Images bindings run locally under Wrangler; B2 requests use the configured +buckets and development keys, so use a separate key and package object prefix. -```sh -npm run packages:setup -npm run packages:dev -``` - -In another terminal: - -```sh -npm run packages:demo -- --fresh -``` - -Setup creates an ignored `.dev.vars.packages-local`, applies local migrations, -and seeds a developer and reviewer. Data lives in `.wrangler/package-demo`; -the API listens on port 8789. The demo uploads a synthetic module, checks private -access, submits and approves it, verifies the public download checksum, and -builds a manifest correction from the stored source. `--fresh` allocates a new -extension ID and includes an SVG icon, so private access is checked before any -publication. It also verifies anonymous job-status denial, owner preview headers, -and public icon access after approval. Without `--fresh`, the original no-icon -fixture is repeatable and reuses artifacts. +`npm run test:all` covers the package lifecycle through the real API routes and +D1 repository, with emulated storage and images. Node tests also exercise package +preparation and B2 storage without a Cloudflare runtime. ## API and review @@ -75,13 +63,15 @@ so it is not uploaded to package storage a second time. The generic `IMessageQueue` producer interface lives in `src/lib/interfaces.ts`; `CloudflareQueueAdapter` implements it. `processPackageJob` depends on a `PackageJobRepository` and a preparation callback, not Cloudflare MessageBatch, -acknowledgements or retry APIs. The D1 repository and Cloudflare consumer wire the -existing storage/image implementation into that lifecycle. Node tests exercise -processing through in-memory implementations without a Cloudflare runtime. -This keeps the new job lifecycle portable; existing D1 and Images integrations -would still need their own adapters for a complete provider migration. - -Production and local configurations each declare a producer/consumer pair. The +acknowledgements or retry APIs. Package preparation accepts a `PackageRepository`, `PackageStore` and +`PackageImageProcessor`. SQL and guarded writes live in `db/package-builds.ts`; +Cloudflare bindings, queue delivery and image processing live under +`src/lib/adapters/cloudflare/`, and the B2 store lives under `src/lib/adapters/b2/`. +The package workflow contains no Cloudflare binding types or SDK calls. A different +runtime supplies these interfaces and its own queue consumer; the rest of the API +still has existing platform integrations outside this package workflow. + +Wrangler declares the production producer/consumer pair; local development emulates it. The consumer handles one message at a time, with concurrency limited to one. Duplicate delivery uses a 20-minute processing lease; a busy duplicate is delayed until that lease can expire. Temporary failures retry after a minute, at most three processing @@ -161,8 +151,7 @@ newer publication. Set these in the integrated local API's ignored `.dev.vars`. The directory worktree's root `.dev.vars` may hold the development credentials, but the API -process must receive them in its own configuration file. The standalone demo -uses `.dev.vars.packages-local`. +process must receive them in its own configuration file. ```dotenv EXTENSION_PACKAGES_S3_ENDPOINT="https://s3.eu-central-003.backblazeb2.com" @@ -176,8 +165,9 @@ EXTENSION_PACKAGES_S3_PREFIX="local" Keep this bucket private. Use a separate development key restricted to the bucket with read, write and delete-file permissions. No public URL or domain is required. Use `EXTENSION_PACKAGES_MODE=local` only outside production. The committed production configuration remains disabled. -Without S3 endpoints, ZIPs and prepared icons use the same local R2 emulator binding. An incomplete S3 configuration -or upstream failure returns 503 rather than silently switching storage backends. +The tests inject a local R2 store when S3 endpoints are unset. Normal local development +uses B2 with development credentials. An incomplete S3 configuration or upstream +failure returns 503 rather than silently switching storage backends. R2 validates the supplied SHA-256 when writing and returns that checksum as object metadata. Its adapter checks this metadata on reads without rehashing the @@ -196,8 +186,7 @@ reservation for retry. ## B2 asset and Cloudflare testing -Add these to the local API's ignored `.dev.vars` (use -`.dev.vars.packages-local` for the standalone API demo): +Add these to the local API's ignored `.dev.vars`: ```dotenv EXTENSION_ASSETS_S3_ENDPOINT="https://s3.eu-central-003.backblazeb2.com" @@ -210,7 +199,7 @@ EXTENSION_ASSETS_PUBLIC_URL="https://extensions-assets.fossbilling.net" Restrict the asset key to this bucket with read, write and delete-file permissions. Pending API previews read these stored objects; cleanup deletes abandoned versions. -An unset endpoint keeps assets in the local emulator instead of B2. Public object keys use +Tests can inject emulated storage when the endpoint is unset. Public object keys use `extensions//icons/.`, for example: ```text diff --git a/src/services/extensions/v2/db/package-builds.ts b/src/services/extensions/v2/db/package-builds.ts new file mode 100644 index 00000000..6881017b --- /dev/null +++ b/src/services/extensions/v2/db/package-builds.ts @@ -0,0 +1,330 @@ +import { v7 as uuidv7 } from "uuid"; +import { ARTIFACT_SELECT } from "./packages"; +import { canonicalJson } from "../packages/manifest"; +import type { Manifest } from "../schemas/manifest"; +import type { + PackageRepository, + PackageOwner, + PackagePublisher, + SourceRow, + ArtifactRow, + IconObject +} from "../packages/repository"; +import { PackageError } from "../packages/errors"; + +export const MAX_PACKAGE_STORAGE_BYTES = 512 * 1024 * 1024; +export const MAX_PACKAGE_OBJECTS = 256; + +function ownerPredicate() { + return `EXISTS (SELECT 1 FROM developers d JOIN users u ON u.id=d.owner_user_id WHERE d.id=? AND d.profile_generation=? AND d.ownership_epoch=? AND d.owner_user_id=? AND u.deleted_at IS NULL AND NOT EXISTS (SELECT 1 FROM extensions e WHERE lower(e.identifier)=lower(?) AND e.developer_id!=d.id) AND (? IS NULL OR EXISTS (SELECT 1 FROM m2m_publisher_clients p WHERE p.client_id=? AND p.developer_id=d.id AND p.ownership_epoch=d.ownership_epoch AND p.revoked_at IS NULL)))`; +} +function ownerParams(owner: PackageOwner, id: string) { + return [ + owner.id, + owner.profile_generation, + owner.ownership_epoch, + owner.owner_user_id, + id, + owner.publisher_client_id ?? null, + owner.publisher_client_id ?? null + ]; +} + +export class PackageBuildDatabase implements PackageRepository { + constructor(private readonly db: D1Database) {} + async getOwner( + userId: string, + extensionId: string, + publisher?: PackagePublisher | null + ) { + const owner = await this.db + .prepare( + `SELECT d.id,d.profile_generation,d.ownership_epoch,d.owner_user_id FROM developers d JOIN users u ON u.id=d.owner_user_id WHERE (? IS NULL AND d.owner_user_id=? OR d.id=? AND d.ownership_epoch=? AND EXISTS (SELECT 1 FROM m2m_publisher_clients p WHERE p.client_id=? AND p.developer_id=d.id AND p.ownership_epoch=d.ownership_epoch AND p.revoked_at IS NULL)) AND u.deleted_at IS NULL AND NOT EXISTS (SELECT 1 FROM extensions e WHERE lower(e.identifier)=lower(?) AND e.developer_id!=d.id)` + ) + .bind( + publisher?.clientId ?? null, + userId, + publisher?.developerId ?? null, + publisher?.ownershipEpoch ?? null, + publisher?.clientId ?? null, + extensionId + ) + .first(); + if (!owner) + throw new PackageError( + "FORBIDDEN", + "An active owning developer profile is required", + 403 + ); + return { ...owner, publisher_client_id: publisher?.clientId }; + } + async findSource(generation: string, identifier: string, version: string) { + return this.db + .prepare( + "SELECT id,sha256,object_id FROM package_sources WHERE profile_generation=? AND extension_identifier=? AND version=?" + ) + .bind(generation, identifier, version) + .first(); + } + async findArtifact(sourceId: string, manifestHash: string) { + return this.db + .prepare(`${ARTIFACT_SELECT} WHERE a.source_id=? AND a.manifest_hash=?`) + .bind(sourceId, manifestHash) + .first(); + } + async hasIcon(sourceId: string) { + return Boolean( + await this.db + .prepare("SELECT 1 FROM package_icons WHERE source_id=?") + .bind(sourceId) + .first() + ); + } + async reserveObject( + owner: PackageOwner, + extensionId: string, + id: string, + bytes: number + ) { + const result = await this.db + .prepare( + `INSERT INTO package_objects (id,created_by,bytes) + SELECT ?,?,? WHERE ${ownerPredicate()} + AND (SELECT count(*) FROM package_objects WHERE created_by=?) < ? + AND (SELECT coalesce(sum(bytes),0) FROM package_objects WHERE created_by=?) + ? <= ?` + ) + .bind( + id, + owner.owner_user_id, + bytes, + ...ownerParams(owner, extensionId), + owner.owner_user_id, + MAX_PACKAGE_OBJECTS, + owner.owner_user_id, + bytes, + MAX_PACKAGE_STORAGE_BYTES + ) + .run(); + if (!result.meta.changes) + throw new PackageError( + "PACKAGE_ADMISSION", + "Ownership changed or package storage quota is exhausted", + 409 + ); + } + async removeObject(id: string) { + await this.db + .prepare("DELETE FROM package_objects WHERE id=?") + .bind(id) + .run(); + } + async claimCleanup() { + const result = await this.db + .prepare( + `UPDATE package_objects SET deleting=1 WHERE id IN ( + SELECT o.id FROM package_objects o WHERE (o.deleting=1 OR o.created_at < datetime('now','-1 day')) + AND NOT EXISTS (SELECT 1 FROM package_jobs j WHERE j.upload_object_id=o.id AND j.status IN ('queued','processing')) + AND NOT EXISTS (SELECT 1 FROM package_sources s WHERE s.object_id=o.id) + AND NOT EXISTS (SELECT 1 FROM package_artifacts a WHERE a.object_id=o.id) + AND NOT EXISTS (SELECT 1 FROM package_icons i WHERE i.avif_object_id=o.id OR i.png_object_id=o.id OR i.svg_object_id=o.id) LIMIT 100 + ) RETURNING id` + ) + .all<{ id: string }>(); + return result.results.map((row) => row.id); + } + async finishCleanup(id: string) { + await this.db + .prepare( + "UPDATE package_jobs SET upload_object_id=NULL WHERE upload_object_id=? AND status IN ('ready','failed')" + ) + .bind(id) + .run(); + await this.db + .prepare("DELETE FROM package_objects WHERE id=? AND deleting=1") + .bind(id) + .run(); + } + async linkSource(owner: PackageOwner, manifest: Manifest, source: SourceRow) { + await this.db + .prepare( + `INSERT INTO package_sources (id,profile_generation,extension_identifier,version,sha256,object_id) + SELECT ?,?,?,?,?,? WHERE ${ownerPredicate()} AND EXISTS (SELECT 1 FROM package_objects WHERE id=? AND deleting=0) + ON CONFLICT DO NOTHING` + ) + .bind( + source.id, + owner.profile_generation, + manifest.id, + manifest.version, + source.sha256, + source.object_id, + ...ownerParams(owner, manifest.id), + source.object_id + ) + .run(); + return await this.db + .prepare( + "SELECT id,sha256,object_id FROM package_sources WHERE profile_generation=? AND extension_identifier=? AND version=?" + ) + .bind(owner.profile_generation, manifest.id, manifest.version) + .first(); + } + async ensureDraft(owner: PackageOwner, identifier: string) { + // Reserve the permanent identity only after ZIP and icon validation succeeds. + await this.db + .prepare( + `INSERT INTO extensions (id,identifier,developer_id,created_by) + SELECT ?,?,?,? WHERE ${ownerPredicate()} ON CONFLICT DO NOTHING` + ) + .bind( + uuidv7(), + identifier, + owner.id, + owner.owner_user_id, + ...ownerParams(owner, identifier) + ) + .run(); + const extension = await this.db + .prepare( + "SELECT id FROM extensions WHERE lower(identifier)=lower(?) AND developer_id=?" + ) + .bind(identifier, owner.id) + .first<{ id: string }>(); + if (!extension) + throw new PackageError( + "FORBIDDEN", + "Extension ownership changed while preparing the package", + 403 + ); + return extension.id; + } + async linkArtifact( + owner: PackageOwner, + manifest: Manifest, + artifact: { + id: string; + sourceId: string; + manifestHash: string; + sha256: string; + objectId: string; + downloadUrl: string; + } + ) { + await this.db + .prepare( + `INSERT INTO package_artifacts (id,source_id,manifest,manifest_hash,sha256,object_id,download_url) + SELECT ?,?,?,?,?,?,? WHERE ${ownerPredicate()} AND EXISTS (SELECT 1 FROM package_objects WHERE id=? AND deleting=0) + ON CONFLICT DO NOTHING` + ) + .bind( + artifact.id, + artifact.sourceId, + canonicalJson(manifest), + artifact.manifestHash, + artifact.sha256, + artifact.objectId, + artifact.downloadUrl, + ...ownerParams(owner, manifest.id), + artifact.objectId + ) + .run(); + return this.findArtifact(artifact.sourceId, artifact.manifestHash); + } + async linkIcon( + owner: PackageOwner, + identifier: string, + sourceId: string, + objects: IconObject[] + ) { + const avif = objects.find((object) => object.format === "avif"); + const png = objects.find((object) => object.format === "png"); + const svg = objects.find((object) => object.format === "svg"); + const result = await this.db + .prepare( + `INSERT INTO package_icons (source_id,avif_object_id,avif_sha256,png_object_id,png_sha256,svg_object_id,svg_sha256) + SELECT ?,?,?,?,?,?,? WHERE ${ownerPredicate()} + AND NOT EXISTS (SELECT 1 FROM package_objects WHERE id IN (?,?,?) AND deleting=1) + ON CONFLICT DO NOTHING` + ) + .bind( + sourceId, + avif?.id ?? null, + avif?.checksum ?? null, + png?.id ?? null, + png?.checksum ?? null, + svg?.id ?? null, + svg?.checksum ?? null, + ...ownerParams(owner, identifier), + avif?.id ?? null, + png?.id ?? null, + svg?.id ?? null + ) + .run(); + return Boolean(result.meta.changes); + } + async getArtifact(id: string) { + return this.db + .prepare(`${ARTIFACT_SELECT} WHERE a.id=?`) + .bind(id) + .first(); + } + async isModerator(userId: string) { + return Boolean( + await this.db + .prepare( + "SELECT 1 FROM users WHERE id=? AND is_moderator=1 AND deleted_at IS NULL" + ) + .bind(userId) + .first() + ); + } + async correctionOwner(userId: string, sourceId: string, manifest: Manifest) { + return this.db + .prepare( + `SELECT d.owner_user_id FROM package_sources s + JOIN developers d ON d.profile_generation=s.profile_generation + JOIN users moderator ON moderator.id=? AND moderator.is_moderator=1 AND moderator.deleted_at IS NULL + WHERE s.id=? AND s.extension_identifier=? AND s.version=? + AND EXISTS (SELECT 1 FROM package_artifacts a WHERE a.source_id=s.id AND a.published_at IS NOT NULL)` + ) + .bind(userId, sourceId, manifest.id, manifest.version) + .first<{ owner_user_id: string }>(); + } + async getArtifactForReader(id: string, userId: string) { + return this.db + .prepare( + `${ARTIFACT_SELECT} WHERE a.id=? AND ( + (a.published_at IS NOT NULL AND EXISTS (SELECT 1 FROM extensions e WHERE lower(e.identifier)=lower(s.extension_identifier) AND e.published_at IS NOT NULL AND e.delisted_at IS NULL)) + OR EXISTS (SELECT 1 FROM users u WHERE u.id=? AND u.deleted_at IS NULL AND (u.is_moderator=1 OR EXISTS (SELECT 1 FROM developers d WHERE d.profile_generation=s.profile_generation AND d.owner_user_id=u.id))) + )` + ) + .bind(id, userId) + .first(); + } + async publishedIcon(identifier: string) { + return this.db + .prepare( + `SELECT e.published_revision_id, coalesce(i.svg_object_id,i.png_object_id) AS object_key + FROM extensions e + JOIN package_artifacts a ON a.download_url=e.download_url AND a.published_at IS NOT NULL + JOIN package_sources s ON s.id=a.source_id AND s.extension_identifier=lower(e.identifier) + JOIN package_icons i ON i.source_id=s.id + WHERE lower(e.identifier)=? AND e.published_at IS NOT NULL AND e.delisted_at IS NULL` + ) + .bind(identifier) + .first<{ published_revision_id: string; object_key: string | null }>(); + } + async updatePublishedIcon( + identifier: string, + revisionId: string, + url: string + ) { + const result = await this.db + .prepare( + "UPDATE extensions SET icon_url=? WHERE lower(identifier)=? AND published_revision_id=? AND delisted_at IS NULL AND icon_url IS NOT ?" + ) + .bind(url, identifier, revisionId, url) + .run(); + return Boolean(result.meta.changes); + } +} diff --git a/src/services/extensions/v2/db/packages.ts b/src/services/extensions/v2/db/packages.ts index 413a85ce..70d775b3 100644 --- a/src/services/extensions/v2/db/packages.ts +++ b/src/services/extensions/v2/db/packages.ts @@ -1,56 +1,10 @@ +import { canonicalJson, contentManifest } from "../packages/manifest"; +import type { ArtifactRow } from "../packages/repository"; import { latestRelease } from "../../../../lib/releases"; -import { ExtensionContent } from "../schemas/extensions"; +import type { ExtensionContent } from "../schemas/extensions"; import { ManifestSchema } from "../schemas/manifest"; import type { DatabaseError } from "../../../../lib/interfaces"; -// Sort nested maps too: object key ordering does not change a manifest. -export function canonicalJson(value: unknown): string { - if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`; - if (value !== null && typeof value === "object") - return `{${Object.entries(value) - .filter(([, v]) => v !== undefined) - .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) - .map(([k, v]) => `${JSON.stringify(k)}:${canonicalJson(v)}`) - .join(",")}}`; - return JSON.stringify(value); -} - -export function contentManifest(id: string, content: ExtensionContent) { - return ManifestSchema.parse({ - id, - manifest_version: content.manifest_version, - type: content.type, - name: content.name, - version: content.version, - description: content.description, - author: content.author, - license: content.license, - homepage_url: content.homepage_url, - requires: content.requires - }); -} - -export interface ArtifactRow { - id: string; - source_id: string; - manifest: string; - sha256: string; - object_id: string; - download_url: string; - published_at: string | null; - created_at: string; - extension_identifier: string; - profile_generation: string; - version: string; - icon_source_id: string | null; - avif_object_id: string | null; - avif_sha256: string | null; - png_object_id: string | null; - png_sha256: string | null; - svg_object_id: string | null; - svg_sha256: string | null; -} - export const ARTIFACT_SELECT = `SELECT a.*, s.extension_identifier, s.profile_generation, s.version, i.source_id AS icon_source_id, i.avif_object_id, i.avif_sha256, i.png_object_id, i.png_sha256, i.svg_object_id, i.svg_sha256 FROM package_artifacts a JOIN package_sources s ON s.id=a.source_id LEFT JOIN package_icons i ON i.source_id=s.id`; diff --git a/src/services/extensions/v2/packages/build.ts b/src/services/extensions/v2/packages/build.ts index 0e05171e..3a6f3a2c 100644 --- a/src/services/extensions/v2/packages/build.ts +++ b/src/services/extensions/v2/packages/build.ts @@ -1,146 +1,43 @@ -import { v7 as uuidv7 } from "uuid"; -import type { PublisherContext } from "../middleware"; -import { sanitizeSvg } from "@zeropress/svg-hush/workerd"; import { buildExtensionPackage, PACKAGE_LIMITS, sha256 } from "../package-builder"; -import { Manifest } from "../schemas/manifest"; -import { ARTIFACT_SELECT, ArtifactRow, canonicalJson } from "../db/packages"; +import type { Manifest } from "../schemas/manifest"; +import { canonicalJson } from "./manifest"; +import type { PackageRepository, PackageOwner, IconObject } from "./repository"; +import type { PackageImageProcessor, IconVariant } from "./images"; import { PackageError } from "./errors"; -import { boundedBytes, PackageStore } from "./store"; - -export interface PackageOwner { - id: string; - profile_generation: string; - ownership_epoch: number; - owner_user_id: string; - publisher_client_id?: string; -} -interface SourceRow { - id: string; - sha256: string; - object_id: string; -} -export const MAX_PACKAGE_STORAGE_BYTES = 512 * 1024 * 1024; -export const MAX_PACKAGE_OBJECTS = 256; - -function ownerPredicate() { - return `EXISTS (SELECT 1 FROM developers d JOIN users u ON u.id=d.owner_user_id WHERE d.id=? AND d.profile_generation=? AND d.ownership_epoch=? AND d.owner_user_id=? AND u.deleted_at IS NULL AND NOT EXISTS (SELECT 1 FROM extensions e WHERE lower(e.identifier)=lower(?) AND e.developer_id!=d.id) AND (? IS NULL OR EXISTS (SELECT 1 FROM m2m_publisher_clients p WHERE p.client_id=? AND p.developer_id=d.id AND p.ownership_epoch=d.ownership_epoch AND p.revoked_at IS NULL)))`; -} -function ownerParams(owner: PackageOwner, id: string) { - return [ - owner.id, - owner.profile_generation, - owner.ownership_epoch, - owner.owner_user_id, - id, - owner.publisher_client_id ?? null, - owner.publisher_client_id ?? null - ]; -} - -export async function packageOwner( - db: D1Database, - userId: string, - extensionId: string, - publisher?: PublisherContext | null -) { - const owner = await db - .prepare( - `SELECT d.id,d.profile_generation,d.ownership_epoch,d.owner_user_id FROM developers d JOIN users u ON u.id=d.owner_user_id WHERE (? IS NULL AND d.owner_user_id=? OR d.id=? AND d.ownership_epoch=? AND EXISTS (SELECT 1 FROM m2m_publisher_clients p WHERE p.client_id=? AND p.developer_id=d.id AND p.ownership_epoch=d.ownership_epoch AND p.revoked_at IS NULL)) AND u.deleted_at IS NULL AND NOT EXISTS (SELECT 1 FROM extensions e WHERE lower(e.identifier)=lower(?) AND e.developer_id!=d.id)` - ) - .bind( - publisher?.clientId ?? null, - userId, - publisher?.developerId ?? null, - publisher?.ownershipEpoch ?? null, - publisher?.clientId ?? null, - extensionId - ) - .first(); - if (!owner) - throw new PackageError( - "FORBIDDEN", - "An active owning developer profile is required", - 403 - ); - return { ...owner, publisher_client_id: publisher?.clientId }; -} +import { boundedBytes, type PackageStore } from "./store"; export async function reserveObject( - db: D1Database, + repository: PackageRepository, store: PackageStore, owner: PackageOwner, - extensionId: string, + identifier: string, bytes: Uint8Array, checksum: string, id: string = crypto.randomUUID() ) { - const result = await db - .prepare( - `INSERT INTO package_objects (id,created_by,bytes) - SELECT ?,?,? WHERE ${ownerPredicate()} - AND (SELECT count(*) FROM package_objects WHERE created_by=?) < ? - AND (SELECT coalesce(sum(bytes),0) FROM package_objects WHERE created_by=?) + ? <= ?` - ) - .bind( - id, - owner.owner_user_id, - bytes.length, - ...ownerParams(owner, extensionId), - owner.owner_user_id, - MAX_PACKAGE_OBJECTS, - owner.owner_user_id, - bytes.length, - MAX_PACKAGE_STORAGE_BYTES - ) - .run(); - if (!result.meta.changes) - throw new PackageError( - "PACKAGE_ADMISSION", - "Ownership changed or package storage quota is exhausted", - 409 - ); - // A reservation is retained on failure/interruption for safe later cleanup. + await repository.reserveObject(owner, identifier, id, bytes.length); + // Failed writes retain their reservation for later cleanup. await store.put(id, bytes, checksum); return id; } - export async function cleanupPackageObjects( - db: D1Database, + repository: PackageRepository, store: PackageStore ) { - const result = await db - .prepare( - `UPDATE package_objects SET deleting=1 WHERE id IN ( - SELECT o.id FROM package_objects o WHERE (o.deleting=1 OR o.created_at < datetime('now','-1 day')) - AND NOT EXISTS (SELECT 1 FROM package_jobs j WHERE j.upload_object_id=o.id AND j.status IN ('queued','processing')) - AND NOT EXISTS (SELECT 1 FROM package_sources s WHERE s.object_id=o.id) - AND NOT EXISTS (SELECT 1 FROM package_artifacts a WHERE a.object_id=o.id) - AND NOT EXISTS (SELECT 1 FROM package_icons i WHERE i.avif_object_id=o.id OR i.png_object_id=o.id OR i.svg_object_id=o.id) LIMIT 100 - ) RETURNING id` - ) - .all<{ id: string }>(); - for (const { id } of result.results) { + const ids = await repository.claimCleanup(); + for (const id of ids) { await store.delete(id); - await db - .prepare( - "UPDATE package_jobs SET upload_object_id=NULL WHERE upload_object_id=? AND status IN ('ready','failed')" - ) - .bind(id) - .run(); - await db - .prepare("DELETE FROM package_objects WHERE id=? AND deleting=1") - .bind(id) - .run(); + await repository.finishCleanup(id); } - return result.results.length; + return ids.length; } export async function buildStoredPackage(input: { - db: D1Database; + repository: PackageRepository; store: PackageStore; owner: PackageOwner; manifest: Manifest; @@ -148,15 +45,14 @@ export async function buildStoredPackage(input: { sourceObjectId?: string; sourceId?: string; baseUrl: string; - images?: ImagesBinding; + images: PackageImageProcessor; }) { - const { db, store, owner, manifest } = input; - let source = await db - .prepare( - "SELECT id,sha256,object_id FROM package_sources WHERE profile_generation=? AND extension_identifier=? AND version=?" - ) - .bind(owner.profile_generation, manifest.id, manifest.version) - .first(); + const { repository, store, owner, manifest } = input; + let source = await repository.findSource( + owner.profile_generation, + manifest.id, + manifest.version + ); if (input.sourceId && source?.id !== input.sourceId) throw new PackageError( "NOT_FOUND", @@ -175,10 +71,7 @@ export async function buildStoredPackage(input: { new Uint8Array(new TextEncoder().encode(manifestJson)) ); if (source) { - const existing = await db - .prepare(`${ARTIFACT_SELECT} WHERE a.source_id=? AND a.manifest_hash=?`) - .bind(source.id, manifestHash) - .first(); + const existing = await repository.findArtifact(source.id, manifestHash); if (existing) return existing; } if (!sourceBytes) { @@ -205,54 +98,27 @@ export async function buildStoredPackage(input: { 422 ); } - const prepared = - source && - (await db - .prepare("SELECT 1 FROM package_icons WHERE source_id=?") - .bind(source.id) - .first()); - const variants = prepared - ? [] - : await optimizePackageIcon( - built.icon, - input.images, - built.iconName === "icon.svg" - ); + const prepared = source && (await repository.hasIcon(source.id)); + const variants = + prepared || !built.icon + ? [] + : await input.images.prepare(built.icon, built.iconName === "icon.svg"); if (!source) { const objectId = input.sourceObjectId ?? (await reserveObject( - db, + repository, store, owner, manifest.id, sourceBytes, built.sourceSha256 )); - const sourceId = crypto.randomUUID(); - await db - .prepare( - `INSERT INTO package_sources (id,profile_generation,extension_identifier,version,sha256,object_id) - SELECT ?,?,?,?,?,? WHERE ${ownerPredicate()} AND EXISTS (SELECT 1 FROM package_objects WHERE id=? AND deleting=0) - ON CONFLICT DO NOTHING` - ) - .bind( - sourceId, - owner.profile_generation, - manifest.id, - manifest.version, - built.sourceSha256, - objectId, - ...ownerParams(owner, manifest.id), - objectId - ) - .run(); - source = await db - .prepare( - "SELECT id,sha256,object_id FROM package_sources WHERE profile_generation=? AND extension_identifier=? AND version=?" - ) - .bind(owner.profile_generation, manifest.id, manifest.version) - .first(); + source = await repository.linkSource(owner, manifest, { + id: crypto.randomUUID(), + sha256: built.sourceSha256, + object_id: objectId + }); if (!source) throw new PackageError( "FORBIDDEN", @@ -262,10 +128,7 @@ export async function buildStoredPackage(input: { if (source.object_id !== objectId) { if (!input.sourceObjectId) { await store.delete(objectId); - await db - .prepare("DELETE FROM package_objects WHERE id=?") - .bind(objectId) - .run(); + await repository.removeObject(objectId); } if (source.sha256 !== built.sourceSha256) throw new PackageError( @@ -275,35 +138,10 @@ export async function buildStoredPackage(input: { ); } } - // Reserve the permanent identity only after ZIP and icon validation succeeds. - await db - .prepare( - `INSERT INTO extensions (id,identifier,developer_id,created_by) - SELECT ?,?,?,? WHERE ${ownerPredicate()} ON CONFLICT DO NOTHING` - ) - .bind( - uuidv7(), - manifest.id, - owner.id, - owner.owner_user_id, - ...ownerParams(owner, manifest.id) - ) - .run(); - const extension = await db - .prepare( - "SELECT id FROM extensions WHERE lower(identifier)=lower(?) AND developer_id=?" - ) - .bind(manifest.id, owner.id) - .first<{ id: string }>(); - if (!extension) - throw new PackageError( - "FORBIDDEN", - "Extension ownership changed while preparing the package", - 403 - ); + const extensionId = await repository.ensureDraft(owner, manifest.id); await storePackageIcon({ - db, - extensionId: extension.id, + repository, + extensionId, store, owner, manifest, @@ -311,7 +149,7 @@ export async function buildStoredPackage(input: { variants }); const objectId = await reserveObject( - db, + repository, store, owner, manifest.id, @@ -320,28 +158,14 @@ export async function buildStoredPackage(input: { ); const artifactId = crypto.randomUUID(); const downloadUrl = `${input.baseUrl}/packages/${artifactId}/download`; - await db - .prepare( - `INSERT INTO package_artifacts (id,source_id,manifest,manifest_hash,sha256,object_id,download_url) - SELECT ?,?,?,?,?,?,? WHERE ${ownerPredicate()} AND EXISTS (SELECT 1 FROM package_objects WHERE id=? AND deleting=0) - ON CONFLICT DO NOTHING` - ) - .bind( - artifactId, - source.id, - manifestJson, - manifestHash, - built.packageSha256, - objectId, - downloadUrl, - ...ownerParams(owner, manifest.id), - objectId - ) - .run(); - const artifact = await db - .prepare(`${ARTIFACT_SELECT} WHERE a.source_id=? AND a.manifest_hash=?`) - .bind(source.id, manifestHash) - .first(); + const artifact = await repository.linkArtifact(owner, manifest, { + id: artifactId, + sourceId: source.id, + manifestHash, + sha256: built.packageSha256, + objectId, + downloadUrl + }); if (!artifact) throw new PackageError( "FORBIDDEN", @@ -350,43 +174,30 @@ export async function buildStoredPackage(input: { ); if (artifact.object_id !== objectId) { await store.delete(objectId); - await db - .prepare("DELETE FROM package_objects WHERE id=?") - .bind(objectId) - .run(); + await repository.removeObject(objectId); } return artifact; } async function storePackageIcon(input: { extensionId: string; - db: D1Database; + repository: PackageRepository; store: PackageStore; owner: PackageOwner; manifest: Manifest; sourceId: string; variants: IconVariant[]; }) { - const { db, store, owner, manifest, sourceId, variants } = input; - if ( - await db - .prepare("SELECT 1 FROM package_icons WHERE source_id=?") - .bind(sourceId) - .first() - ) - return; + const { repository, store, owner, manifest, sourceId, variants } = input; + if (await repository.hasIcon(sourceId)) return; const assetId = crypto.randomUUID(); - const objects: { - format: IconVariant["format"]; - id: string; - checksum: string; - }[] = []; + const objects: IconObject[] = []; for (const { format, bytes } of variants) { const checksum = await sha256(bytes); objects.push({ format, id: await reserveObject( - db, + repository, store, owner, manifest.id, @@ -397,44 +208,18 @@ async function storePackageIcon(input: { checksum }); } - const avif = objects.find((object) => object.format === "avif"); - const png = objects.find((object) => object.format === "png"); - const svg = objects.find((object) => object.format === "svg"); - const result = await db - .prepare( - `INSERT INTO package_icons (source_id,avif_object_id,avif_sha256,png_object_id,png_sha256,svg_object_id,svg_sha256) - SELECT ?,?,?,?,?,?,? WHERE ${ownerPredicate()} - AND NOT EXISTS (SELECT 1 FROM package_objects WHERE id IN (?,?,?) AND deleting=1) - ON CONFLICT DO NOTHING` - ) - .bind( - sourceId, - avif?.id ?? null, - avif?.checksum ?? null, - png?.id ?? null, - png?.checksum ?? null, - svg?.id ?? null, - svg?.checksum ?? null, - ...ownerParams(owner, manifest.id), - avif?.id ?? null, - png?.id ?? null, - svg?.id ?? null - ) - .run(); - if (!result.meta.changes) { + const linked = await repository.linkIcon( + owner, + manifest.id, + sourceId, + objects + ); + if (!linked) { for (const object of objects) { await store.delete(object.id); - await db - .prepare("DELETE FROM package_objects WHERE id=?") - .bind(object.id) - .run(); + await repository.removeObject(object.id); } - if ( - !(await db - .prepare("SELECT 1 FROM package_icons WHERE source_id=?") - .bind(sourceId) - .first()) - ) + if (!(await repository.hasIcon(sourceId))) throw new PackageError( "FORBIDDEN", "Ownership changed while processing icon", @@ -442,60 +227,3 @@ async function storePackageIcon(input: { ); } } - -type IconVariant = { - format: "avif" | "png" | "svg"; - bytes: Uint8Array; -}; -async function optimizePackageIcon( - icon: Uint8Array | null, - images?: ImagesBinding, - isSvg = false -): Promise { - if (!icon) return []; - if (isSvg) { - try { - return [{ format: "svg", bytes: new Uint8Array(sanitizeSvg(icon)) }]; - } catch { - throw new PackageError( - "INVALID_ICON", - "Package SVG icon could not be sanitized", - 422 - ); - } - } - if (!images) - throw new PackageError( - "PACKAGES_DISABLED", - "Image processing is not configured", - 503 - ); - const variants: IconVariant[] = []; - try { - const info = await images.info(new Blob([icon]).stream()); - // Check decoded format too: a misleading filename must not bypass sanitization. - if (info.format === "image/svg+xml") - return [{ format: "svg", bytes: new Uint8Array(sanitizeSvg(icon)) }]; - for (const format of ["avif", "png"] as const) { - const output = await images - .input(new Blob([icon]).stream()) - .transform({ - width: 256, - height: 256, - fit: "scale-down" - }) - .output({ format: `image/${format}`, anim: false }); - variants.push({ - format, - bytes: await boundedBytes(output.image(), 1024 * 1024) - }); - } - } catch { - throw new PackageError( - "INVALID_ICON", - "Package icon could not be processed", - 422 - ); - } - return variants; -} diff --git a/src/services/extensions/v2/packages/cloudflare-queue.ts b/src/services/extensions/v2/packages/cloudflare-queue.ts deleted file mode 100644 index fb502a95..00000000 --- a/src/services/extensions/v2/packages/cloudflare-queue.ts +++ /dev/null @@ -1,125 +0,0 @@ -import { logError } from "../../../../lib/logger"; -import { createCloudflareBindings } from "../../../../lib/adapters/cloudflare"; -import { CloudflareQueueAdapter } from "../../../../lib/adapters/cloudflare/queue"; -import { PackageJobsDatabase } from "../db/package-jobs"; -import { buildStoredPackage, packageOwner } from "./build"; -import { PackageError } from "./errors"; -import { boundedBytes, configuredPackageStore } from "./store"; -import { processPackageJob, dispatchPackageJob, type PackageJob } from "./jobs"; -import { PACKAGE_LIMITS } from "../package-builder"; - -export async function prepareQueuedPackage( - env: CloudflareBindings, - job: PackageJob -): Promise { - const bindings = createCloudflareBindings(env); - const getEnv = (name: string) => bindings.environment.get(name); - const store = configuredPackageStore(env, getEnv); - if (!store) - throw new PackageError( - "PACKAGES_DISABLED", - "Package storage is not configured", - 503 - ); - const previous = job.owner; - const current = await packageOwner( - env.DB_EXTENSIONS, - previous.owner_user_id, - job.manifest.id, - previous.publisher_client_id - ? { - clientId: previous.publisher_client_id, - developerId: previous.id, - ownershipEpoch: previous.ownership_epoch, - canPublish: false - } - : null - ); - if ( - current.id !== previous.id || - current.profile_generation !== previous.profile_generation || - current.ownership_epoch !== previous.ownership_epoch - ) - throw new PackageError( - "FORBIDDEN", - "Package ownership changed while queued", - 403 - ); - let source: Uint8Array | undefined; - if (job.uploadObjectId) { - const body = await store.get( - job.uploadObjectId, - job.uploadSha256 ?? undefined - ); - if (!body) - throw new PackageError( - "SOURCE_UNAVAILABLE", - "Queued source ZIP is unavailable", - 503 - ); - source = await boundedBytes(body, PACKAGE_LIMITS.sourceBytes); - } - const artifact = await buildStoredPackage({ - db: env.DB_EXTENSIONS, - store, - owner: current, - manifest: job.manifest, - source, - sourceObjectId: job.uploadObjectId ?? undefined, - sourceId: job.sourceId ?? undefined, - baseUrl: getEnv("EXTENSION_PACKAGES_BASE_URL")!.replace(/\/$/, ""), - images: env.PACKAGE_IMAGES - }); - return artifact.id; -} - -export async function consumePackageJobs( - batch: MessageBatch<{ jobId: string }>, - env: CloudflareBindings -) { - const jobs = new PackageJobsDatabase(env.DB_EXTENSIONS); - for (const message of batch.messages) { - if ( - !message.body || - typeof message.body.jobId !== "string" || - !/^[a-f0-9-]{36}$/.test(message.body.jobId) - ) { - message.ack(); - continue; - } - try { - const result = await processPackageJob(jobs, message.body.jobId, (job) => - prepareQueuedPackage(env, job) - ); - if (result === "busy") message.retry({ delaySeconds: 1200 }); - else if (result === "retry") message.retry({ delaySeconds: 60 }); - else message.ack(); - } catch { - message.retry({ delaySeconds: 60 }); - } - } -} - -export async function recoverPackageJobs(env: CloudflareBindings) { - const mode = createCloudflareBindings(env).environment.get( - "EXTENSION_PACKAGES_MODE" - ); - if ( - !env.EXTENSION_PACKAGE_QUEUE || - !["local", "enabled"].includes(mode ?? "") - ) - return; - const jobs = new PackageJobsDatabase(env.DB_EXTENSIONS); - await jobs.expire(); - const queue = new CloudflareQueueAdapter(env.EXTENSION_PACKAGE_QUEUE); - for (const id of await jobs.pending()) { - try { - await dispatchPackageJob(jobs, queue, id); - } catch (error) { - logError("extensions/packages", "Package job dispatch failed", { - jobId: id, - error: error instanceof Error ? error.message : String(error) - }); - } - } -} diff --git a/src/services/extensions/v2/packages/images.ts b/src/services/extensions/v2/packages/images.ts new file mode 100644 index 00000000..efc41174 --- /dev/null +++ b/src/services/extensions/v2/packages/images.ts @@ -0,0 +1,10 @@ +export interface IconVariant { + format: "avif" | "png" | "svg"; + bytes: Uint8Array; +} +export interface PackageImageProcessor { + prepare( + icon: Uint8Array, + isSvg: boolean + ): Promise; +} diff --git a/src/services/extensions/v2/packages/jobs.ts b/src/services/extensions/v2/packages/jobs.ts index 99cbecec..0af883d0 100644 --- a/src/services/extensions/v2/packages/jobs.ts +++ b/src/services/extensions/v2/packages/jobs.ts @@ -1,7 +1,7 @@ import { logError } from "../../../../lib/logger"; import type { IMessageQueue } from "../../../../lib/interfaces"; import type { Manifest } from "../schemas/manifest"; -import type { PackageOwner } from "./build"; +import type { PackageOwner } from "./repository"; import { PackageError } from "./errors"; export interface PackageJob { diff --git a/src/services/extensions/v2/packages/manifest.ts b/src/services/extensions/v2/packages/manifest.ts new file mode 100644 index 00000000..b0d36b92 --- /dev/null +++ b/src/services/extensions/v2/packages/manifest.ts @@ -0,0 +1,29 @@ +import type { ExtensionContent } from "../schemas/extensions"; +import { ManifestSchema } from "../schemas/manifest"; + +// Sort nested maps too: object key ordering does not change a manifest. +export function canonicalJson(value: unknown): string { + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`; + if (value !== null && typeof value === "object") + return `{${Object.entries(value) + .filter(([, v]) => v !== undefined) + .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) + .map(([k, v]) => `${JSON.stringify(k)}:${canonicalJson(v)}`) + .join(",")}}`; + return JSON.stringify(value); +} + +export function contentManifest(id: string, content: ExtensionContent) { + return ManifestSchema.parse({ + id, + manifest_version: content.manifest_version, + type: content.type, + name: content.name, + version: content.version, + description: content.description, + author: content.author, + license: content.license, + homepage_url: content.homepage_url, + requires: content.requires + }); +} diff --git a/src/services/extensions/v2/packages/prepare.ts b/src/services/extensions/v2/packages/prepare.ts new file mode 100644 index 00000000..42bb791c --- /dev/null +++ b/src/services/extensions/v2/packages/prepare.ts @@ -0,0 +1,70 @@ +import type { PackageRepository } from "./repository"; +import type { PackageImageProcessor } from "./images"; +import type { PackageStore } from "./store"; +import type { PackageJob } from "./jobs"; +import { PackageError } from "./errors"; +import { boundedBytes } from "./store"; +import { buildStoredPackage } from "./build"; +import { PACKAGE_LIMITS } from "../package-builder"; + +export interface PackagePreparation { + repository: PackageRepository; + store: PackageStore; + images: PackageImageProcessor; + baseUrl: string; +} + +export async function prepareQueuedPackage( + services: PackagePreparation, + job: PackageJob +): Promise { + const { repository, store } = services; + const previous = job.owner; + const current = await repository.getOwner( + previous.owner_user_id, + job.manifest.id, + previous.publisher_client_id + ? { + clientId: previous.publisher_client_id, + developerId: previous.id, + ownershipEpoch: previous.ownership_epoch + } + : null + ); + if ( + current.id !== previous.id || + current.profile_generation !== previous.profile_generation || + current.ownership_epoch !== previous.ownership_epoch + ) + throw new PackageError( + "FORBIDDEN", + "Package ownership changed while queued", + 403 + ); + let source: Uint8Array | undefined; + if (job.uploadObjectId) { + const body = await store.get( + job.uploadObjectId, + job.uploadSha256 ?? undefined + ); + if (!body) + throw new PackageError( + "SOURCE_UNAVAILABLE", + "Queued source ZIP is unavailable", + 503 + ); + source = await boundedBytes(body, PACKAGE_LIMITS.sourceBytes); + } + const artifact = await buildStoredPackage({ + repository, + store, + owner: current, + manifest: job.manifest, + source, + sourceObjectId: job.uploadObjectId ?? undefined, + sourceId: job.sourceId ?? undefined, + baseUrl: services.baseUrl.replace(/\/$/, ""), + images: services.images + }); + return artifact.id; +} diff --git a/src/services/extensions/v2/packages/repository.ts b/src/services/extensions/v2/packages/repository.ts new file mode 100644 index 00000000..a43e097c --- /dev/null +++ b/src/services/extensions/v2/packages/repository.ts @@ -0,0 +1,95 @@ +import type { Manifest } from "../schemas/manifest"; + +export interface PackageOwner { + id: string; + profile_generation: string; + ownership_epoch: number; + owner_user_id: string; + publisher_client_id?: string; +} +export interface SourceRow { + id: string; + sha256: string; + object_id: string; +} +export interface ArtifactRow { + id: string; + source_id: string; + manifest: string; + sha256: string; + object_id: string; + download_url: string; + published_at: string | null; + created_at: string; + extension_identifier: string; + profile_generation: string; + version: string; + icon_source_id: string | null; + avif_object_id: string | null; + avif_sha256: string | null; + png_object_id: string | null; + png_sha256: string | null; + svg_object_id: string | null; + svg_sha256: string | null; +} + +export interface PackagePublisher { + clientId: string; + developerId: string; + ownershipEpoch: number; +} +export interface IconObject { + format: "svg" | "avif" | "png"; + id: string; + checksum: string; +} +export interface PackageRepository { + getOwner( + userId: string, + identifier: string, + publisher?: PackagePublisher | null + ): Promise; + findSource( + generation: string, + identifier: string, + version: string + ): Promise; + findArtifact( + sourceId: string, + manifestHash: string + ): Promise; + hasIcon(sourceId: string): Promise; + reserveObject( + owner: PackageOwner, + identifier: string, + id: string, + bytes: number + ): Promise; + removeObject(id: string): Promise; + claimCleanup(): Promise; + finishCleanup(id: string): Promise; + linkSource( + owner: PackageOwner, + manifest: Manifest, + source: SourceRow + ): Promise; + ensureDraft(owner: PackageOwner, identifier: string): Promise; + linkArtifact( + owner: PackageOwner, + manifest: Manifest, + artifact: { + id: string; + sourceId: string; + manifestHash: string; + sha256: string; + objectId: string; + downloadUrl: string; + } + ): Promise; + linkIcon( + owner: PackageOwner, + identifier: string, + sourceId: string, + objects: IconObject[] + ): Promise; +} diff --git a/src/services/extensions/v2/packages/store.ts b/src/services/extensions/v2/packages/store.ts index 9eb4e899..5f5901fd 100644 --- a/src/services/extensions/v2/packages/store.ts +++ b/src/services/extensions/v2/packages/store.ts @@ -1,7 +1,4 @@ import { PackageError } from "./errors"; -import type { Context } from "hono"; -import { b2ObjectStore } from "./b2-store"; -import { getPlatform } from "../../../../lib/middleware"; export interface PackageStore { put( @@ -16,122 +13,6 @@ export interface PackageStore { delete(key: string): Promise; } -export function packageStore( - c: Context<{ Bindings: CloudflareBindings }> -): PackageStore | null { - return configuredPackageStore(c.env, (name) => getPlatform(c).getEnv(name)); -} - -export function configuredPackageStore( - env: CloudflareBindings, - getEnv: (name: string) => string | undefined -): PackageStore | null { - const mode = getEnv("EXTENSION_PACKAGES_MODE"); - const production = getEnv("ENVIRONMENT") === "production"; - if (mode !== "enabled" && !(mode === "local" && !production)) return null; - if (production) { - const base = getEnv("EXTENSION_PACKAGES_BASE_URL"); - let url: URL; - try { - url = new URL(base ?? ""); - } catch { - throw new PackageError( - "PACKAGES_DISABLED", - "Package download URL is not configured", - 503 - ); - } - if ( - url.protocol !== "https:" || - url.username || - url.password || - url.search || - url.hash || - url.pathname !== "/extensions/v2" || - !getEnv("EXTENSION_PACKAGES_S3_ENDPOINT") || - !getEnv("EXTENSION_PACKAGES_S3_PREFIX") || - getEnv("EXTENSION_PACKAGES_S3_PREFIX") === "local" || - !getEnv("EXTENSION_ASSETS_S3_ENDPOINT") || - !getEnv("EXTENSION_ASSETS_PUBLIC_URL") || - !env.PACKAGE_IMAGES - ) { - throw new PackageError( - "PACKAGES_DISABLED", - "Production package storage, images or download URL is not configured", - 503 - ); - } - } - const packages = getEnv("EXTENSION_PACKAGES_S3_ENDPOINT") - ? b2ObjectStore(getEnv, "packages") - : null; - const assets = getEnv("EXTENSION_ASSETS_S3_ENDPOINT") - ? b2ObjectStore(getEnv, "assets") - : null; - const bucket = env.EXTENSION_PACKAGE_BUCKET; - if (!packages && !bucket) return null; - const localBucket = () => { - if (!bucket) - throw new PackageError( - "PACKAGES_DISABLED", - "Storage is not configured", - 503 - ); - return bucket; - }; - const remoteFor = (key: string) => - key.startsWith("extensions/") ? assets : packages; - return { - async put(key, bytes, checksum) { - const remote = remoteFor(key); - if (remote) return remote.put(key, bytes, checksum); - const result = await localBucket().put(key, bytes, { - onlyIf: { etagDoesNotMatch: "*" }, - sha256: checksum, - httpMetadata: { - contentType: key.endsWith(".avif") - ? "image/avif" - : key.endsWith(".png") - ? "image/png" - : key.endsWith(".svg") - ? "image/svg+xml" - : "application/zip", - cacheControl: "private, no-store" - } - }); - if (!result) throw new Error("Package object already exists"); - }, - async get(key, checksum) { - const remote = remoteFor(key); - if (remote) return remote.get(key, checksum); - const object = await localBucket().get(key); - if (!object) return null; - if (checksum) { - const actual = object.checksums.sha256; - const hex = actual - ? Array.from(new Uint8Array(actual), (byte) => - byte.toString(16).padStart(2, "0") - ).join("") - : null; - if (hex !== checksum) { - await object.body.cancel(); - throw new PackageError( - "PACKAGE_UNAVAILABLE", - "Stored object failed checksum verification", - 503 - ); - } - } - return object.body; - }, - async delete(key) { - const remote = remoteFor(key); - if (remote) return remote.delete(key); - await localBucket().delete(key); - } - }; -} - export async function boundedBytes( body: ReadableStream | null, limit: number diff --git a/src/services/extensions/v2/routes/moderation.ts b/src/services/extensions/v2/routes/moderation.ts index 5fb41f94..ebdde3a3 100644 --- a/src/services/extensions/v2/routes/moderation.ts +++ b/src/services/extensions/v2/routes/moderation.ts @@ -1,4 +1,4 @@ -import { schedulePublishedIconUpdate } from "../packages/public-assets"; +import { schedulePublishedIconUpdate } from "../../../../lib/adapters/cloudflare/package-assets"; import { paceContentAccount, requireApproveAccess, diff --git a/src/services/extensions/v2/routes/packages.ts b/src/services/extensions/v2/routes/packages.ts index cbc0aed2..1cdacdd9 100644 --- a/src/services/extensions/v2/routes/packages.ts +++ b/src/services/extensions/v2/routes/packages.ts @@ -5,7 +5,9 @@ import { createRoute, z } from "@hono/zod-openapi"; import type { Context, MiddlewareHandler } from "hono"; import { getAuth } from "../../../../lib/auth"; import { getPlatform } from "../../../../lib/middleware"; -import { ARTIFACT_SELECT, ArtifactRow, canonicalJson } from "../db/packages"; +import { canonicalJson } from "../packages/manifest"; +import type { ArtifactRow } from "../packages/repository"; +import { PackageBuildDatabase } from "../db/package-builds"; import { requireContentWrite, getPublisher, @@ -15,13 +17,10 @@ import { paceContentAccount, paceContentIp } from "../middleware"; -import { - reserveObject, - packageOwner, - cleanupPackageObjects -} from "../packages/build"; +import { reserveObject, cleanupPackageObjects } from "../packages/build"; import { PackageError } from "../packages/errors"; -import { boundedBytes, packageStore } from "../packages/store"; +import { boundedBytes } from "../packages/store"; +import { packageStore } from "../../../../lib/adapters/cloudflare/package-store"; import { ManifestSchema } from "../schemas/manifest"; import { PackageArtifactSchema, @@ -65,10 +64,7 @@ function artifactResponse(row: ArtifactRow) { } async function jobResponse(db: D1Database, job: PackageJob) { const artifact = job.artifactId - ? await db - .prepare(`${ARTIFACT_SELECT} WHERE a.id=?`) - .bind(job.artifactId) - .first() + ? await new PackageBuildDatabase(db).getArtifact(job.artifactId) : null; return { result: { @@ -188,26 +184,19 @@ export function registerPackageRoutes(app: ExtensionsV2App) { 413 ); const db = c.env.DB_EXTENSIONS; + const repository = new PackageBuildDatabase(db); const publisher = getPublisher(c); let ownerUserId = getAuth(c).userId; // Moderator corrections can rebuild published source, but cannot upload new files for another developer. if (form.source_id && !publisher) { - const correction = await db - .prepare( - ` - SELECT d.owner_user_id FROM package_sources s - JOIN developers d ON d.profile_generation=s.profile_generation - JOIN users moderator ON moderator.id=? AND moderator.is_moderator=1 AND moderator.deleted_at IS NULL - WHERE s.id=? AND s.extension_identifier=? AND s.version=? - AND EXISTS (SELECT 1 FROM package_artifacts a WHERE a.source_id=s.id AND a.published_at IS NOT NULL) - ` - ) - .bind(ownerUserId, form.source_id, manifest.id, manifest.version) - .first<{ owner_user_id: string }>(); + const correction = await repository.correctionOwner( + ownerUserId, + form.source_id, + manifest + ); if (correction) ownerUserId = correction.owner_user_id; } - const owner = await packageOwner( - db, + const owner = await repository.getOwner( ownerUserId, manifest.id, publisher @@ -239,7 +228,7 @@ export function registerPackageRoutes(app: ExtensionsV2App) { if (!job) { const uploadObjectId = source ? await reserveObject( - db, + repository, store, owner, manifest.id, @@ -298,22 +287,17 @@ export function registerPackageRoutes(app: ExtensionsV2App) { async (c) => { try { const db = c.env.DB_EXTENSIONS; + const repository = new PackageBuildDatabase(db); const job = await new PackageJobsDatabase(db).get( c.req.valid("param").jobId ); if (!job) throw new PackageError("NOT_FOUND", "Package job not found", 404); const userId = getAuth(c).userId; - const moderator = await db - .prepare( - "SELECT 1 FROM users WHERE id=? AND is_moderator=1 AND deleted_at IS NULL" - ) - .bind(userId) - .first(); + const moderator = await repository.isModerator(userId); if (!moderator) { const publisher = getPublisher(c); - const owner = await packageOwner( - db, + const owner = await repository.getOwner( userId, job.manifest.id, publisher @@ -362,7 +346,7 @@ export function registerPackageRoutes(app: ExtensionsV2App) { { result: { deleted: await cleanupPackageObjects( - c.env.DB_EXTENSIONS, + new PackageBuildDatabase(c.env.DB_EXTENSIONS), packageStore(c)! ) } @@ -432,14 +416,9 @@ export function registerPackageRoutes(app: ExtensionsV2App) { async (c: Context<{ Bindings: CloudflareBindings }>) => { try { const auth = getOptionalAuth(c); - const row = await c.env.DB_EXTENSIONS.prepare( - `${ARTIFACT_SELECT} WHERE a.id=? AND ( - (a.published_at IS NOT NULL AND EXISTS (SELECT 1 FROM extensions e WHERE lower(e.identifier)=lower(s.extension_identifier) AND e.published_at IS NOT NULL AND e.delisted_at IS NULL)) - OR EXISTS (SELECT 1 FROM users u WHERE u.id=? AND u.deleted_at IS NULL AND (u.is_moderator=1 OR EXISTS (SELECT 1 FROM developers d WHERE d.profile_generation=s.profile_generation AND d.owner_user_id=u.id))) - )` - ) - .bind(c.req.param("packageId"), auth?.userId ?? "") - .first(); + const row = await new PackageBuildDatabase( + c.env.DB_EXTENSIONS + ).getArtifactForReader(c.req.param("packageId")!, auth?.userId ?? ""); if (!row) throw new PackageError("NOT_FOUND", "Package not found", 404); if (!download && !icon) return c.json(artifactResponse(row), 200); diff --git a/test/env.d.ts b/test/env.d.ts index 48c5a596..8d6cd677 100644 --- a/test/env.d.ts +++ b/test/env.d.ts @@ -11,6 +11,7 @@ declare module "cloudflare:test" { // hence augmenting this namespace directly rather than the module above. declare namespace Cloudflare { interface Env { + EXTENSION_PACKAGE_BUCKET: R2Bucket; TEST_MIGRATIONS_EXTENSIONS: import("cloudflare:test").D1Migration[]; TEST_MIGRATIONS_CENTRAL_ALERTS: import("cloudflare:test").D1Migration[]; } diff --git a/test/integration/scheduled.test.ts b/test/integration/scheduled.test.ts index 1bf44bb9..0a17d647 100644 --- a/test/integration/scheduled.test.ts +++ b/test/integration/scheduled.test.ts @@ -5,7 +5,7 @@ const mocks = vi.hoisted(() => ({ maintain: vi.fn(), report: vi.fn() })); -vi.mock("../../src/services/extensions/v2/packages/cloudflare-queue", () => ({ +vi.mock("../../src/lib/adapters/cloudflare/package-jobs", () => ({ consumePackageJobs: vi.fn(), recoverPackageJobs: mocks.recover })); diff --git a/test/services/extensions/v2/b2-store.test.ts b/test/services/extensions/v2/b2-store.node.test.ts similarity index 98% rename from test/services/extensions/v2/b2-store.test.ts rename to test/services/extensions/v2/b2-store.node.test.ts index f8f0f547..90aedd2f 100644 --- a/test/services/extensions/v2/b2-store.test.ts +++ b/test/services/extensions/v2/b2-store.node.test.ts @@ -1,5 +1,5 @@ import { afterEach, describe, expect, it, vi } from "vitest"; -import { b2ObjectStore } from "../../../../src/services/extensions/v2/packages/b2-store"; +import { b2ObjectStore } from "../../../../src/lib/adapters/b2/package-store"; import { sha256, PACKAGE_LIMITS diff --git a/test/services/extensions/v2/package-build.node.test.ts b/test/services/extensions/v2/package-build.node.test.ts new file mode 100644 index 00000000..97806995 --- /dev/null +++ b/test/services/extensions/v2/package-build.node.test.ts @@ -0,0 +1,174 @@ +import { describe, expect, it, vi } from "vitest"; +import { + TextReader, + TextWriter, + Uint8ArrayReader, + Uint8ArrayWriter, + ZipReader, + ZipWriter +} from "@zip.js/zip.js"; +import { prepareQueuedPackage } from "../../../../src/services/extensions/v2/packages/prepare"; +import type { + PackageRepository, + ArtifactRow, + SourceRow, + IconObject +} from "../../../../src/services/extensions/v2/packages/repository"; +import type { PackageJob } from "../../../../src/services/extensions/v2/packages/jobs"; +import type { PackageStore } from "../../../../src/services/extensions/v2/packages/store"; +import { sha256 } from "../../../../src/services/extensions/v2/package-builder"; + +const zipOptions = { useWebWorkers: false, useCompressionStream: true }; +describe("package preparation outside Cloudflare", () => { + it("prepares and corrects ZIPs with injected persistence, storage and image processing", async () => { + const manifest = { + manifest_version: 1 as const, + type: "module" as const, + id: "node-example", + name: "Node Example", + version: "1.0.0", + description: "Prepared in Node", + author: { name: "Developer" }, + license: "MIT", + requires: { fossbilling: ">=0.8.0" } + }; + const owner = { + id: crypto.randomUUID(), + profile_generation: crypto.randomUUID(), + ownership_epoch: 1, + owner_user_id: "owner" + }; + const extensionId = "019a0000-0000-7000-8000-000000000001"; + let source: SourceRow | null = null; + let icons: IconObject[] | null = null; + const artifacts = new Map(); + const unexpected = async (): Promise => { + throw new Error("Unexpected cleanup in normal preparation"); + }; + const repository: PackageRepository = { + getOwner: async () => owner, + findSource: async () => source, + findArtifact: async (_sourceId, hash) => artifacts.get(hash) ?? null, + hasIcon: async () => icons !== null, + reserveObject: async () => {}, + removeObject: unexpected, + claimCleanup: unexpected, + finishCleanup: unexpected, + linkSource: async (_owner, _manifest, input) => (source ??= input), + ensureDraft: async () => extensionId, + linkIcon: async (_owner, _identifier, _sourceId, input) => { + icons = input; + return true; + }, + linkArtifact: async (_owner, metadata, input) => { + const svg = icons?.find((icon) => icon.format === "svg"); + const row: ArtifactRow = { + id: input.id, + source_id: input.sourceId, + manifest: JSON.stringify(metadata), + sha256: input.sha256, + object_id: input.objectId, + download_url: input.downloadUrl, + published_at: null, + created_at: new Date().toISOString(), + extension_identifier: metadata.id, + profile_generation: owner.profile_generation, + version: metadata.version, + icon_source_id: input.sourceId, + svg_object_id: svg?.id ?? null, + svg_sha256: svg?.checksum ?? null, + avif_object_id: null, + avif_sha256: null, + png_object_id: null, + png_sha256: null + }; + artifacts.set(input.manifestHash, row); + return row; + } + }; + const objects = new Map>(); + const store: PackageStore = { + put: async (key, bytes) => { + objects.set(key, new Uint8Array(bytes)); + }, + get: async (key) => + objects.has(key) ? new Blob([objects.get(key)!]).stream() : null, + delete: async (key) => { + objects.delete(key); + } + }; + const images = { + prepare: vi.fn(async (bytes: Uint8Array) => [ + { format: "svg" as const, bytes } + ]) + }; + const writer = new ZipWriter(new Uint8ArrayWriter(), zipOptions); + await writer.add( + "Service.php", + new TextReader("' + ) + ); + const upload = await writer.close(); + const uploadId = crypto.randomUUID(); + objects.set(uploadId, upload); + const job: PackageJob = { + id: crypto.randomUUID(), + fingerprint: "node", + owner, + manifest, + sourceId: null, + uploadObjectId: uploadId, + uploadSha256: await sha256(upload), + status: "processing", + attempts: 1, + artifactId: null, + error: null, + createdAt: "now", + updatedAt: "now" + }; + const services = { + repository, + store, + images, + baseUrl: "https://directory.example.test/extensions/v2" + }; + const id = await prepareQueuedPackage(services, job); + const first = [...artifacts.values()].find((row) => row.id === id)!; + const bytes = objects.get(first.object_id)!; + expect(await sha256(bytes)).toBe(first.sha256); + const reader = new ZipReader(new Uint8ArrayReader(bytes), zipOptions); + try { + const files: Record = {}; + for (const entry of await reader.getEntries()) { + if (!entry.directory) + files[entry.filename] = await entry.getData!(new TextWriter()); + } + expect(JSON.parse(files["manifest.json"])).toEqual(manifest); + expect(files["Service.php"]).toBe(" row.id === correctedId + )!; + expect(corrected.source_id).toBe(first.source_id); + expect(corrected.svg_object_id).toBe(first.svg_object_id); + expect(corrected.sha256).not.toBe(first.sha256); + expect(images.prepare).toHaveBeenCalledOnce(); + }); +}); diff --git a/test/services/extensions/v2/packages.test.ts b/test/services/extensions/v2/packages.test.ts index 7ad4f4a1..03631106 100644 --- a/test/services/extensions/v2/packages.test.ts +++ b/test/services/extensions/v2/packages.test.ts @@ -2,8 +2,9 @@ import { PackageJobsDatabase } from "../../../../src/services/extensions/v2/db/p import { processPackageJob } from "../../../../src/services/extensions/v2/packages/jobs"; import { recoverPackageJobs, - prepareQueuedPackage -} from "../../../../src/services/extensions/v2/packages/cloudflare-queue"; + packagePreparation +} from "../../../../src/lib/adapters/cloudflare/package-jobs"; +import { prepareQueuedPackage } from "../../../../src/services/extensions/v2/packages/prepare"; import { beforeEach, afterEach, describe, expect, it, vi } from "vitest"; import { env } from "cloudflare:workers"; import { @@ -27,10 +28,8 @@ import { import { PackageArtifactSchema } from "../../../../src/services/extensions/v2/schemas/packages"; import { ManifestSchema } from "../../../../src/services/extensions/v2/schemas/manifest"; import { ExtensionContentSchema } from "../../../../src/services/extensions/v2/schemas/extensions"; -import { - validatePackageContent, - contentManifest -} from "../../../../src/services/extensions/v2/db/packages"; +import { validatePackageContent } from "../../../../src/services/extensions/v2/db/packages"; +import { contentManifest } from "../../../../src/services/extensions/v2/packages/manifest"; import { sha256 } from "../../../../src/services/extensions/v2/package-builder"; import { setupExtensionsV2Tests, @@ -61,11 +60,13 @@ beforeEach(async () => { vi.spyOn(env.EXTENSION_PACKAGE_QUEUE, "send").mockResolvedValue({ metadata: { metrics: { backlogCount: 0, backlogBytes: 0 } } }); - delete env.EXTENSION_PACKAGES_S3_ENDPOINT; - delete env.EXTENSION_ASSETS_S3_ENDPOINT; - env.ENVIRONMENT = "local"; - env.EXTENSION_PACKAGES_MODE = "local"; - env.EXTENSION_PACKAGES_BASE_URL = "http://localhost:8789/extensions/v2"; + Reflect.deleteProperty(env, "EXTENSION_PACKAGES_S3_ENDPOINT"); + Reflect.deleteProperty(env, "EXTENSION_ASSETS_S3_ENDPOINT"); + Object.assign(env, { + ENVIRONMENT: "local", + EXTENSION_PACKAGES_MODE: "local", + EXTENSION_PACKAGES_BASE_URL: "http://localhost:8789/extensions/v2" + }); await authHeaders("owner"); await seedDeveloper("developer", "owner"); }); @@ -137,7 +138,7 @@ async function finishJob(response: Response): Promise { attempt++ ) { await processPackageJob(jobs, result.id, (job) => - prepareQueuedPackage(env, job) + prepareQueuedPackage(packagePreparation(env), job) ); job = (await jobs.get(result.id))!; if (job.status === "processing") @@ -519,7 +520,7 @@ describe("queued package preparation", () => { await bucket().delete(job.uploadObjectId!); expect( await processPackageJob(jobs, job.id, (job) => - prepareQueuedPackage(env, job) + prepareQueuedPackage(packagePreparation(env), job) ) ).toBe("retry"); expect(await jobs.get(job.id)).toMatchObject({ @@ -532,7 +533,7 @@ describe("queued package preparation", () => { }); expect( await processPackageJob(jobs, job.id, (job) => - prepareQueuedPackage(env, job) + prepareQueuedPackage(packagePreparation(env), job) ) ).toBe("ack"); expect(await jobs.get(job.id)).toMatchObject({ @@ -570,7 +571,10 @@ describe("queued package preparation", () => { false ); expect((await jobs.get(job.id))!.status).toBe("processing"); - const artifactId = await prepareQueuedPackage(env, claimed!); + const artifactId = await prepareQueuedPackage( + packagePreparation(env), + claimed! + ); await jobs.complete(job.id, "second", artifactId); expect((await jobs.get(job.id))!.status).toBe("ready"); }); @@ -589,7 +593,7 @@ describe("queued package preparation", () => { const prepare = vi.fn( ( job: import("../../../../src/services/extensions/v2/packages/jobs").PackageJob - ) => prepareQueuedPackage(env, job) + ) => prepareQueuedPackage(packagePreparation(env), job) ); await Promise.all([ processPackageJob(jobs, job.id, prepare), @@ -1588,8 +1592,8 @@ describe("stored directory packages", () => { ).toBe(true); Object.assign(env, { EXTENSION_PACKAGES_S3_PREFIX: "local" }); expect((await upload()).status).toBe(503); - delete env.EXTENSION_PACKAGES_S3_PREFIX; - env.ENVIRONMENT = "local"; + Reflect.deleteProperty(env, "EXTENSION_PACKAGES_S3_PREFIX"); + Object.assign(env, { ENVIRONMENT: "local" }); }); it("checks account ownership, upload bounds, invalid archives and disabled storage", async () => { diff --git a/vitest.node.config.ts b/vitest.node.config.ts index 2a7b7a6c..11e93c0b 100644 --- a/vitest.node.config.ts +++ b/vitest.node.config.ts @@ -6,7 +6,7 @@ export default defineConfig({ include: [ "test/lib/adapters/node/**/*.test.ts", "test/services/extensions/v2/migrations.test.ts", - "test/services/extensions/v2/package-jobs.node.test.ts" + "test/services/extensions/v2/*.node.test.ts" ], testTimeout: 10000, diff --git a/worker-configuration.d.ts b/worker-configuration.d.ts index 2d9ed9da..5e494216 100644 --- a/worker-configuration.d.ts +++ b/worker-configuration.d.ts @@ -1,75 +1,46 @@ /* eslint-disable */ -// Generated by Wrangler by running `wrangler types --env-interface=CloudflareBindings` (hash: 9b9b2dbaff5d17401395576e365b39a4) +// Generated by Wrangler by running `wrangler types --env-interface=CloudflareBindings` (hash: 060e6ec948699f0972680fc6251f3e8f) // Runtime types generated with workerd@1.20261103.1 2026-10-01 interface __BaseEnv_CloudflareBindings { AUTH_KV: KVNamespace; CACHE_KV: KVNamespace; DOWNLOAD_BUCKET: R2Bucket; - EXTENSION_PACKAGE_BUCKET?: R2Bucket; DB_CENTRAL_ALERTS: D1Database; DB_EXTENSIONS: D1Database; EXTENSION_PACKAGE_QUEUE: Queue; EXTENSION_WRITE_RATE_LIMITER: RateLimit; PROFILE_CREATION_RATE_LIMITER: RateLimit; PACKAGE_IMAGES: ImagesBinding; - ENVIRONMENT: "local" | "production"; + ENVIRONMENT: "production"; + EXTENSION_PACKAGES_MODE: "disabled"; + EXTENSION_PACKAGES_BASE_URL: "https://api.fossbilling.net/extensions/v2"; + EXTENSION_PACKAGES_S3_ENDPOINT: "https://s3.eu-central-003.backblazeb2.com"; + EXTENSION_PACKAGES_S3_REGION: "eu-central-003"; + EXTENSION_PACKAGES_S3_BUCKET: "fb-extensions-packages"; + EXTENSION_PACKAGES_S3_PREFIX: "production"; + EXTENSION_ASSETS_S3_ENDPOINT: "https://s3.eu-central-003.backblazeb2.com"; + EXTENSION_ASSETS_S3_REGION: "eu-central-003"; + EXTENSION_ASSETS_S3_BUCKET: "fb-extensions-assets"; + EXTENSION_ASSETS_PUBLIC_URL: "https://extensions-assets.fossbilling.net"; EXTENSIONS_RETENTION_MODE: "dry-run"; + EXTENSIONS_V2_EMAIL_PROVIDER: "mxroute"; + EXTENSIONS_V2_MXROUTE_SERVER: "wednesday.mxrouting.net"; + EXTENSIONS_V2_MXROUTE_USERNAME: "extensions@fossbilling.org"; M2M_ISSUER: "https://auth.fossbilling.net"; M2M_JWKS_URL: "https://auth.fossbilling.net/.well-known/jwks.json"; M2M_AUDIENCE: "https://api.fossbilling.net"; - EXTENSIONS_V2_EMAIL_PROVIDER: "disabled" | "mxroute"; - EXTENSIONS_V2_MXROUTE_SERVER: "" | "wednesday.mxrouting.net"; - EXTENSIONS_V2_MXROUTE_USERNAME: "" | "extensions@fossbilling.org"; - EXTENSION_PACKAGES_MODE: "local" | "disabled"; - EXTENSION_PACKAGES_BASE_URL: "http://localhost:8789/extensions/v2" | "https://api.fossbilling.net/extensions/v2"; GITHUB_TOKEN: string; ASSERTION_SIGNING_SECRET: string; EXTENSIONS_V2_MXROUTE_PASSWORD: string; EXTENSIONS_REVALIDATE_SECRET: string; PREVIEW_GITHUB_BUDGET: DurableObjectNamespace; EXTENSIONS_FRONTEND: Fetcher /* extensions */; - EXTENSION_PACKAGES_S3_ENDPOINT?: "https://s3.eu-central-003.backblazeb2.com"; - EXTENSION_PACKAGES_S3_REGION?: "eu-central-003"; - EXTENSION_PACKAGES_S3_BUCKET?: "fb-extensions-packages"; - EXTENSION_PACKAGES_S3_PREFIX?: "production"; - EXTENSION_ASSETS_S3_ENDPOINT?: "https://s3.eu-central-003.backblazeb2.com"; - EXTENSION_ASSETS_S3_REGION?: "eu-central-003"; - EXTENSION_ASSETS_S3_BUCKET?: "fb-extensions-assets"; - EXTENSION_ASSETS_PUBLIC_URL?: "https://extensions-assets.fossbilling.net"; } declare namespace Cloudflare { interface GlobalProps { mainModule: typeof import("./src/app/index"); durableNamespaces: "PreviewGitHubBudget"; } - interface PackagesLocalEnv { - AUTH_KV: KVNamespace; - CACHE_KV: KVNamespace; - DOWNLOAD_BUCKET: R2Bucket; - EXTENSION_PACKAGE_BUCKET: R2Bucket; - DB_CENTRAL_ALERTS: D1Database; - DB_EXTENSIONS: D1Database; - EXTENSION_PACKAGE_QUEUE: Queue; - EXTENSION_WRITE_RATE_LIMITER: RateLimit; - PROFILE_CREATION_RATE_LIMITER: RateLimit; - PACKAGE_IMAGES: ImagesBinding; - ENVIRONMENT: "local"; - EXTENSIONS_RETENTION_MODE: "dry-run"; - M2M_ISSUER: "https://auth.fossbilling.net"; - M2M_JWKS_URL: "https://auth.fossbilling.net/.well-known/jwks.json"; - M2M_AUDIENCE: "https://api.fossbilling.net"; - EXTENSIONS_V2_EMAIL_PROVIDER: "disabled"; - EXTENSIONS_V2_MXROUTE_SERVER: ""; - EXTENSIONS_V2_MXROUTE_USERNAME: ""; - EXTENSION_PACKAGES_MODE: "local"; - EXTENSION_PACKAGES_BASE_URL: "http://localhost:8789/extensions/v2"; - GITHUB_TOKEN: string; - ASSERTION_SIGNING_SECRET: string; - EXTENSIONS_V2_MXROUTE_PASSWORD: string; - EXTENSIONS_REVALIDATE_SECRET: string; - PREVIEW_GITHUB_BUDGET: DurableObjectNamespace; - EXTENSIONS_FRONTEND: Fetcher /* extensions */; - } interface Env extends __BaseEnv_CloudflareBindings {} } interface CloudflareBindings extends __BaseEnv_CloudflareBindings {} @@ -77,7 +48,7 @@ type StringifyValues> = { [Binding in keyof EnvType]: EnvType[Binding] extends string ? EnvType[Binding] : string; }; declare namespace NodeJS { - interface ProcessEnv extends StringifyValues> {} + interface ProcessEnv extends StringifyValues> {} } declare module "*.txt" { const value: string; diff --git a/wrangler.jsonc b/wrangler.jsonc index f8c5f1b2..43776f37 100644 --- a/wrangler.jsonc +++ b/wrangler.jsonc @@ -127,86 +127,5 @@ "period": 60 } } - ], - "env": { - "packages-local": { - "queues": { - "producers": [ - { - "binding": "EXTENSION_PACKAGE_QUEUE", - "queue": "extension-packages-local" - } - ], - "consumers": [ - { - "queue": "extension-packages-local", - "max_batch_size": 1, - "max_batch_timeout": 1, - "max_retries": 5, - "max_concurrency": 1 - } - ] - }, - "images": { "binding": "PACKAGE_IMAGES", "remote": false }, - "routes": [], - "vars": { - "ENVIRONMENT": "local", - "EXTENSIONS_RETENTION_MODE": "dry-run", - "M2M_ISSUER": "https://auth.fossbilling.net", - "M2M_JWKS_URL": "https://auth.fossbilling.net/.well-known/jwks.json", - "M2M_AUDIENCE": "https://api.fossbilling.net", - "EXTENSIONS_V2_EMAIL_PROVIDER": "disabled", - "EXTENSIONS_V2_MXROUTE_SERVER": "", - "EXTENSIONS_V2_MXROUTE_USERNAME": "", - "EXTENSION_PACKAGES_MODE": "local", - "EXTENSION_PACKAGES_BASE_URL": "http://localhost:8789/extensions/v2" - }, - "d1_databases": [ - { - "binding": "DB_CENTRAL_ALERTS", - "database_id": "00000000-0000-0000-0000-000000000001", - "migrations_dir": "src/services/central-alerts/v1/db/migrations" - }, - { - "binding": "DB_EXTENSIONS", - "database_id": "00000000-0000-0000-0000-000000000002", - "migrations_dir": "src/services/extensions/v2/db/migrations" - } - ], - "kv_namespaces": [ - { "binding": "AUTH_KV", "id": "00000000000000000000000000000001" }, - { "binding": "CACHE_KV", "id": "00000000000000000000000000000002" } - ], - "r2_buckets": [ - { "binding": "DOWNLOAD_BUCKET", "bucket_name": "downloads-local" }, - { - "binding": "EXTENSION_PACKAGE_BUCKET", - "bucket_name": "extension-packages-local" - } - ], - "durable_objects": { - "bindings": [ - { - "name": "PREVIEW_GITHUB_BUDGET", - "class_name": "PreviewGitHubBudget" - } - ] - }, - "services": [ - { "binding": "EXTENSIONS_FRONTEND", "service": "extensions" } - ], - "ratelimits": [ - { - "name": "EXTENSION_WRITE_RATE_LIMITER", - "namespace_id": "1001", - "simple": { "limit": 50, "period": 60 } - }, - { - "name": "PROFILE_CREATION_RATE_LIMITER", - "namespace_id": "1002", - "simple": { "limit": 3, "period": 60 } - } - ] - } - } + ] } From 7470f59fc535f8886a1d32625660760bbd5c076c Mon Sep 17 00:00:00 2001 From: Adam Daley Date: Mon, 9 Nov 2026 08:29:19 +0000 Subject: [PATCH 08/12] Tighten package regression tests and remove the timing spike --- src/services/extensions/v2/PACKAGING.md | 4 +- src/services/extensions/v2/package-builder.ts | 2 - .../extensions/v2/package-builder.test.ts | 47 ++++-------- test/services/extensions/v2/packages.test.ts | 74 ++++++++++++------- 4 files changed, 63 insertions(+), 64 deletions(-) diff --git a/src/services/extensions/v2/PACKAGING.md b/src/services/extensions/v2/PACKAGING.md index 02c90238..76f68ae2 100644 --- a/src/services/extensions/v2/PACKAGING.md +++ b/src/services/extensions/v2/PACKAGING.md @@ -3,7 +3,9 @@ Authors upload ready-to-install distribution ZIPs, with any compilation already complete. The API validates and repacks them with directory-generated manifests, then publishes them through the existing revision review. It never runs uploaded -code or package build scripts. Package storage uses the private B2 bucket when configured, otherwise Wrangler's local R2 emulator. Prepared icons use the public asset bucket, including icons for pending submissions. +code or package build scripts. ZIPs use the private B2 bucket; prepared icons use +the public asset bucket, including icons for pending submissions. Automated tests +inject emulated storage. Package routes are disabled by default. Local development and production enablement are separate modes. ## Local development and testing diff --git a/src/services/extensions/v2/package-builder.ts b/src/services/extensions/v2/package-builder.ts index ea1c0439..59aa497d 100644 --- a/src/services/extensions/v2/package-builder.ts +++ b/src/services/extensions/v2/package-builder.ts @@ -203,8 +203,6 @@ export async function buildExtensionPackage( manifest, sourceSha256: await sha256(source), packageSha256: await sha256(bytes), - sourceFileCount: files.length - 1, - expandedBytes, icon: icons[0]?.icon ?? null, iconName: icons[0]?.path }; diff --git a/test/services/extensions/v2/package-builder.test.ts b/test/services/extensions/v2/package-builder.test.ts index 8e8eef87..e3dc87a8 100644 --- a/test/services/extensions/v2/package-builder.test.ts +++ b/test/services/extensions/v2/package-builder.test.ts @@ -49,7 +49,7 @@ async function contents(bytes: Uint8Array) { } } -describe("local directory package builder", () => { +describe("directory package builder", () => { it("rejects overlong and deeply nested paths before walking ancestors", async () => { for (const path of [ "a/".repeat(20000) + "file.php", @@ -61,10 +61,9 @@ describe("local directory package builder", () => { ).rejects.toThrow(/path exceeds/); } const path = "a/".repeat(PACKAGE_LIMITS.pathDepth - 1) + "file.php"; - expect( - (await buildExtensionPackage(await zip([[path, "file"]]), manifest)) - .sourceFileCount - ).toBe(1); + await expect( + buildExtensionPackage(await zip([[path, "file"]]), manifest) + ).resolves.toMatchObject({ manifest }); }); it("replaces the manifest and preserves every source file", async () => { const source = await zip([ @@ -78,7 +77,6 @@ describe("local directory package builder", () => { "Service.php": "" }); - expect(built.sourceFileCount).toBe(2); expect(built.sourceSha256).toBe(await sha256(source)); expect(built.packageSha256).toBe(await sha256(built.bytes)); const repeated = await buildExtensionPackage(source, manifest); @@ -240,9 +238,13 @@ describe("local directory package builder", () => { it.each(["manifest.json", "file.php"])( "checks CRCs before copying or replacing %s", async (filename) => { - const source = await zip([[filename, "unique-contents"]], { - level: 0 - }); + const source = await zip( + [ + [filename, "unique-contents"], + ["Service.php", " @@ -250,7 +252,9 @@ describe("local directory package builder", () => { ); expect(offset).toBeGreaterThan(0); source[offset] ^= 1; - await expect(buildExtensionPackage(source, manifest)).rejects.toThrow(); + await expect(buildExtensionPackage(source, manifest)).rejects.toThrow( + "Invalid CRC32" + ); } ); @@ -315,27 +319,4 @@ describe("local directory package builder", () => { buildExtensionPackage(await zip([["manifest.json", "{}"]]), manifest) ).rejects.toThrow("no source files"); }); - - it("builds and reads back a representative multi-file package in Workers", async () => { - const source = await zip( - Array.from({ length: 100 }, (_, i) => [ - `src/file-${i}.php`, - " { await new Promise((resolve) => setTimeout(resolve, 5)); } if (job.status === "ready") { - const read = await get( + return get( `${root}/packages/${job.artifactId}`, await authHeaders("owner") ); - if (read.status !== 200) return read; - return new Response(await read.text(), { - status: 201, - headers: { "Content-Type": "application/json" } - }); } return Response.json( { error: job.error }, @@ -167,7 +163,7 @@ async function built( ) { const response = await upload(metadata, source, sourceId); const body = (await response.json()) as { result: unknown }; - expect(response.status, JSON.stringify(body)).toBe(201); + expect(response.status, JSON.stringify(body)).toBe(200); return PackageArtifactSchema.parse(body.result); } function content(artifact: ReturnType) { @@ -269,7 +265,7 @@ describe("publisher package builds", () => { }); expect( (await finishJob(Response.json({ result: job }, { status: 202 }))).status - ).toBe(201); + ).toBe(200); const ready = await get(`${root}/package-jobs/${job.id}`, headers); expect(ready.status).toBe(200); expect(await ready.json()).toMatchObject({ @@ -289,7 +285,7 @@ describe("publisher package builds", () => { }); it("builds under the mapped developer while retaining owner privacy", async () => { const response = await machineBuild(); - expect(response.status).toBe(201); + expect(response.status).toBe(200); const artifact = PackageArtifactSchema.parse( ((await response.json()) as { result: unknown }).result ); @@ -428,10 +424,20 @@ describe("queued package preparation", () => { ) ).status ).toBe(403); - const finished = await finishJob( - Response.json({ result: job }, { status: 202 }) - ); - expect(finished.status).toBe(201); + const ctx = createExecutionContext(); + const batch = createMessageBatch("extension-packages", [ + { + id: "delivery", + timestamp: new Date(), + attempts: 1, + body: { jobId: job.id } + } + ]); + await app.queue(batch, env); + expect(await getQueueResult(batch, ctx)).toMatchObject({ + explicitAcks: ["delivery"], + retryMessages: [] + }); const status = await get( `${root}/package-jobs/${job.id}`, await authHeaders("owner") @@ -928,7 +934,7 @@ describe("stored directory packages", () => { artifact.source_id, "moderator" ); - expect(response.status).toBe(201); + expect(response.status).toBe(200); const next = PackageArtifactSchema.parse( ((await response.json()) as { result: unknown }).result ); @@ -962,7 +968,7 @@ describe("stored directory packages", () => { expect( await db.prepare("SELECT count(*) AS count FROM package_sources").first() ).toEqual({ count: 0 }); - expect((await upload()).status).toBe(201); + expect((await upload()).status).toBe(200); }); function configureB2(overrides: Record = {}) { @@ -1225,7 +1231,7 @@ describe("stored directory packages", () => { ); const changed = { ...manifest(), version: "1.1.0" }; expect((await upload(changed, await zip("changed source"))).status).toBe( - 201 + 200 ); await seedDeveloper("other-developer", "other"); expect( @@ -1394,19 +1400,31 @@ describe("stored directory packages", () => { artifact_id: crypto.randomUUID() })) ]; + const headers = await authHeaders("owner"); const prepared = vi.spyOn(db, "prepare"); - const error = await validatePackageContent( - db, - "package-demo", - "developer", - ExtensionContentSchema.parse({ ...content(artifact), releases }) - ); - expect(error?.message).toBe("Release does not match its package artifact"); + const response = await post(`${root}/extensions`, headers, { + identifier: "package-demo", + ...content(artifact), + releases + }); const artifactQueries = prepared.mock.calls .map(([sql]) => sql) .filter((sql) => sql.includes("FROM package_artifacts")); - expect(artifactQueries).toHaveLength(2); // publication state + the complete artifact set - expect(artifactQueries[1]).toContain("json_each(?)"); + prepared.mockRestore(); + expect(response.status).toBe(409); + expect(await response.json()).toMatchObject({ + error: { message: "Release does not match its package artifact" } + }); + // Keep validation bounded rather than querying once per release. + expect(artifactQueries.length).toBeLessThanOrEqual(2); + expect( + ( + await post(`${root}/extensions`, headers, { + identifier: "package-demo", + ...content(artifact) + }) + ).status + ).toBe(201); }); it("revalidates the artifact at approval rather than trusting submission-time checks", async () => { const artifact = await built(); @@ -1628,7 +1646,7 @@ describe("stored directory packages", () => { expect( await db.prepare("SELECT count(*) AS count FROM package_objects").first() ).toEqual({ count: 1 }); - expect((await upload()).status).toBe(201); + expect((await upload()).status).toBe(200); expect( await db.prepare("SELECT count(*) AS count FROM package_sources").first() ).toEqual({ count: 1 }); @@ -1668,6 +1686,6 @@ describe("stored directory packages", () => { artifact.source_id ) ).status - ).toBe(201); + ).toBe(200); }); }); From 906135a5b103b0462754542d28daf35e02c6d32f Mon Sep 17 00:00:00 2001 From: Adam Daley Date: Mon, 9 Nov 2026 09:00:08 +0000 Subject: [PATCH 09/12] Keep operational configuration in dashboard runtime variables --- README.md | 16 +++++++++-- src/services/extensions/v2/PACKAGING.md | 38 ++++++++++++++++--------- worker-configuration.d.ts | 13 ++------- wrangler.jsonc | 10 +------ 4 files changed, 42 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index 8019d258..0f4186fb 100644 --- a/README.md +++ b/README.md @@ -76,6 +76,17 @@ We use [Cloudflare D1](https://developers.cloudflare.com/d1/) and [KV](https://d ### Environment Variables +Production B2 connection details and MXroute server/username are managed in the +Worker dashboard under **Settings > Variables and Secrets** as runtime variables. +Credentials are stored as secrets. Build environment variables do not configure +the running Worker. `keep_vars: true` preserves dashboard-managed variables on +Wrangler deployments; keep their values out of the committed `vars` map and docs. +Public URLs and deployment controls remain in `wrangler.jsonc`. + +For local development, put operational values and credentials in the ignored +`.dev.vars` file. Document variable names and placeholders, not production values. +Package storage configuration is listed in [PACKAGING.md](src/services/extensions/v2/PACKAGING.md). + - `GITHUB_TOKEN`: A GitHub Personal Access Token (classic) with public repo read access. - `ASSERTION_SIGNING_SECRET`: Shared HMAC secret used to verify the short-lived bearer assertions minted by the Extensions site. Configure the same value in both Workers; it is never sent to clients. - `ASSERTION_SIGNING_SECRET_PREVIOUS`: Optional previous HMAC secret accepted during a signing-key rotation. @@ -83,7 +94,8 @@ We use [Cloudflare D1](https://developers.cloudflare.com/d1/) and [KV](https://d - `EXTENSIONS_V2_EMAIL_PROVIDER`: `mxroute` or `resend` to send moderator notification emails, or unset/`disabled` to skip sending. - `EXTENSIONS_V2_EMAIL_FROM`: Sender address for notification emails (default `noreply@fossbilling.org`). - `EXTENSIONS_V2_EMAIL_REPLY_TO`: Optional Reply-To address for a monitored inbox. Omit when unset (no Reply-To header is sent) — notification emails do not invite replies. -- `EXTENSIONS_V2_MXROUTE_SERVER`, `EXTENSIONS_V2_MXROUTE_USERNAME`, `EXTENSIONS_V2_MXROUTE_PASSWORD`: Mailbox credentials for the MXroute SMTP API (`https://smtpapi.mxroute.com/`). Required when `EXTENSIONS_V2_EMAIL_PROVIDER=mxroute`. +- `EXTENSIONS_V2_MXROUTE_SERVER`, `EXTENSIONS_V2_MXROUTE_USERNAME`: Dashboard-managed runtime variables identifying the mailbox for the MXroute SMTP API (`https://smtpapi.mxroute.com/`). Required when `EXTENSIONS_V2_EMAIL_PROVIDER=mxroute`. +- `EXTENSIONS_V2_MXROUTE_PASSWORD`: Mailbox password, stored as a secret. Required when `EXTENSIONS_V2_EMAIL_PROVIDER=mxroute`. - `EXTENSIONS_V2_RESEND_API_KEY`: API key for Resend. Required when `EXTENSIONS_V2_EMAIL_PROVIDER=resend`. - `EXTENSIONS_RETENTION_MODE`: `dry-run` (report only) or `compact` (prune old revision bodies). See [extensions v2 README](src/services/extensions/v2/README.md#revisions-and-retention). - `M2M_ISSUER`, `M2M_JWKS_URL`, `M2M_AUDIENCE`: Issuer, JWKS endpoint, and expected audience for machine-to-machine access tokens minted by the central auth service (`FOSSBilling/auth`). Defaults live in `wrangler.jsonc`; preview environments need them pointed at the issuing preview. See [extensions v2 README](src/services/extensions/v2/README.md#machine-publishing). @@ -100,7 +112,7 @@ npm install ### Local Setup -1. Create a `.dev.vars` file for your secrets: +1. Create a `.dev.vars` file for local runtime variables and secrets: ```env GITHUB_TOKEN="your-token" diff --git a/src/services/extensions/v2/PACKAGING.md b/src/services/extensions/v2/PACKAGING.md index 76f68ae2..31abf6e9 100644 --- a/src/services/extensions/v2/PACKAGING.md +++ b/src/services/extensions/v2/PACKAGING.md @@ -122,7 +122,7 @@ conflicts and corrupt archives are rejected. Checksums cover the final stored ZI Include at most one root `icon.svg`, `icon.png`, `icon.webp`, or `icon.avif`. There is no manifest icon field; packages without an icon use the generic icon. -Packaging prepares display images once and stores them in `fb-extensions-assets`. +Packaging prepares display images once and stores them in the configured public asset bucket. The original icon stays unchanged in the ZIP. A new extension receives its permanent UUIDv7 draft identity after package validation, before the first icon is stored. The first review submission reuses that draft identity. @@ -156,9 +156,9 @@ worktree's root `.dev.vars` may hold the development credentials, but the API process must receive them in its own configuration file. ```dotenv -EXTENSION_PACKAGES_S3_ENDPOINT="https://s3.eu-central-003.backblazeb2.com" -EXTENSION_PACKAGES_S3_REGION="eu-central-003" -EXTENSION_PACKAGES_S3_BUCKET="fb-extensions-packages" +EXTENSION_PACKAGES_S3_ENDPOINT="https://s3..backblazeb2.com" +EXTENSION_PACKAGES_S3_REGION="" +EXTENSION_PACKAGES_S3_BUCKET="your-development-packages-bucket" EXTENSION_PACKAGES_S3_ACCESS_KEY_ID="your-keyID" EXTENSION_PACKAGES_S3_SECRET_ACCESS_KEY="your-applicationKey" EXTENSION_PACKAGES_S3_PREFIX="local" @@ -191,9 +191,9 @@ reservation for retry. Add these to the local API's ignored `.dev.vars`: ```dotenv -EXTENSION_ASSETS_S3_ENDPOINT="https://s3.eu-central-003.backblazeb2.com" -EXTENSION_ASSETS_S3_REGION="eu-central-003" -EXTENSION_ASSETS_S3_BUCKET="fb-extensions-assets" +EXTENSION_ASSETS_S3_ENDPOINT="https://s3..backblazeb2.com" +EXTENSION_ASSETS_S3_REGION="" +EXTENSION_ASSETS_S3_BUCKET="your-development-assets-bucket" EXTENSION_ASSETS_S3_ACCESS_KEY_ID="your-keyID" EXTENSION_ASSETS_S3_SECRET_ACCESS_KEY="your-applicationKey" EXTENSION_ASSETS_PUBLIC_URL="https://extensions-assets.fossbilling.net" @@ -233,9 +233,21 @@ Keep B2 daily caps and alerts configured for download and transaction charges. ## Production configuration (prepared locally) -The top-level Wrangler configuration names the production resources but keeps -`EXTENSION_PACKAGES_MODE=disabled`. No production resources or secrets are -created by a local build or type-generation command. +The top-level Wrangler configuration declares bindings, public URLs and deployment +controls, keeping `EXTENSION_PACKAGES_MODE=disabled`. B2 endpoints, regions, bucket +names and the package prefix are dashboard-managed runtime variables. Configure +these under the Worker's **Settings > Variables and Secrets**, not build settings: + +| Runtime variables | Purpose | +| ------------------------------------------------------------------------------------------------ | -------------------------------------------------------------------- | +| `EXTENSION_PACKAGES_S3_ENDPOINT`, `EXTENSION_PACKAGES_S3_REGION`, `EXTENSION_PACKAGES_S3_BUCKET` | Private ZIP storage connection | +| `EXTENSION_PACKAGES_S3_PREFIX` | ZIP object prefix; production requires an explicit non-`local` value | +| `EXTENSION_ASSETS_S3_ENDPOINT`, `EXTENSION_ASSETS_S3_REGION`, `EXTENSION_ASSETS_S3_BUCKET` | Public image storage connection | + +`keep_vars: true` preserves these values during Wrangler deployments. Do not +reintroduce them in the committed `vars` map. Keep development values in ignored +`.dev.vars` files; the examples above use placeholders. No production resources +or secrets are created by a local build or type-generation command. Before deployment (including an automatic deployment after merge): @@ -245,8 +257,8 @@ Before deployment (including an automatic deployment after merge): application routes, not deployment-time binding validation. 2. Supply `EXTENSION_PACKAGES_S3_ACCESS_KEY_ID` and `EXTENSION_PACKAGES_S3_SECRET_ACCESS_KEY` as Worker secrets for - `fb-extensions-packages`. Supply separate `EXTENSION_ASSETS_S3_ACCESS_KEY_ID` - and `EXTENSION_ASSETS_S3_SECRET_ACCESS_KEY` for `fb-extensions-assets`. + the private package bucket. Supply separate `EXTENSION_ASSETS_S3_ACCESS_KEY_ID` + and `EXTENSION_ASSETS_S3_SECRET_ACCESS_KEY` for the public asset bucket. Development keys in `.dev.vars` are not production secrets. 3. Confirm the Images binding is enabled for the account, the scoped CDN security headers remain configured, and B2 caps/alerts are in place. @@ -260,7 +272,7 @@ Before deployment (including an automatic deployment after merge): Production requires HTTPS download URLs with the `/extensions/v2` base path, complete credentials for both B2 buckets, an explicit non-`local` ZIP object prefix, a public asset URL and the Images binding. An incomplete configuration fails with 503; -it cannot fall back to local ZIP storage. Private package objects use the `production` prefix in this configuration. +it cannot fall back to local ZIP storage. Private package objects use the configured prefix. Public image objects use the UUID-based layout described above. The disabled mode blocks package reads as well as writes. Once managed releases diff --git a/worker-configuration.d.ts b/worker-configuration.d.ts index 2d571fae..5e25d541 100644 --- a/worker-configuration.d.ts +++ b/worker-configuration.d.ts @@ -1,5 +1,5 @@ /* eslint-disable */ -// Generated by Wrangler by running `wrangler types --env-interface=CloudflareBindings` (hash: 060e6ec948699f0972680fc6251f3e8f) +// Generated by Wrangler by running `wrangler types --env-interface=CloudflareBindings` (hash: e610663ea57a18e0636f4e14dece74ae) // Runtime types generated with workerd@1.20261104.1 2026-10-01 interface __BaseEnv_CloudflareBindings { AUTH_KV: KVNamespace; @@ -14,18 +14,9 @@ interface __BaseEnv_CloudflareBindings { ENVIRONMENT: "production"; EXTENSION_PACKAGES_MODE: "disabled"; EXTENSION_PACKAGES_BASE_URL: "https://api.fossbilling.net/extensions/v2"; - EXTENSION_PACKAGES_S3_ENDPOINT: "https://s3.eu-central-003.backblazeb2.com"; - EXTENSION_PACKAGES_S3_REGION: "eu-central-003"; - EXTENSION_PACKAGES_S3_BUCKET: "fb-extensions-packages"; - EXTENSION_PACKAGES_S3_PREFIX: "production"; - EXTENSION_ASSETS_S3_ENDPOINT: "https://s3.eu-central-003.backblazeb2.com"; - EXTENSION_ASSETS_S3_REGION: "eu-central-003"; - EXTENSION_ASSETS_S3_BUCKET: "fb-extensions-assets"; EXTENSION_ASSETS_PUBLIC_URL: "https://extensions-assets.fossbilling.net"; EXTENSIONS_RETENTION_MODE: "dry-run"; EXTENSIONS_V2_EMAIL_PROVIDER: "mxroute"; - EXTENSIONS_V2_MXROUTE_SERVER: "wednesday.mxrouting.net"; - EXTENSIONS_V2_MXROUTE_USERNAME: "extensions@fossbilling.org"; M2M_ISSUER: "https://auth.fossbilling.net"; M2M_JWKS_URL: "https://auth.fossbilling.net/.well-known/jwks.json"; M2M_AUDIENCE: "https://api.fossbilling.net"; @@ -48,7 +39,7 @@ type StringifyValues> = { [Binding in keyof EnvType]: EnvType[Binding] extends string ? EnvType[Binding] : string; }; declare namespace NodeJS { - interface ProcessEnv extends StringifyValues> {} + interface ProcessEnv extends StringifyValues> {} } declare module "*.txt" { const value: string; diff --git a/wrangler.jsonc b/wrangler.jsonc index 43776f37..9ea8ef5c 100644 --- a/wrangler.jsonc +++ b/wrangler.jsonc @@ -20,24 +20,16 @@ "mode": "smart" }, "preview_urls": true, + // Preserve operational runtime variables managed in the Worker dashboard. "keep_vars": true, "vars": { "ENVIRONMENT": "production", // Enable only after the queue, Images binding and B2 secrets are configured. "EXTENSION_PACKAGES_MODE": "disabled", "EXTENSION_PACKAGES_BASE_URL": "https://api.fossbilling.net/extensions/v2", - "EXTENSION_PACKAGES_S3_ENDPOINT": "https://s3.eu-central-003.backblazeb2.com", - "EXTENSION_PACKAGES_S3_REGION": "eu-central-003", - "EXTENSION_PACKAGES_S3_BUCKET": "fb-extensions-packages", - "EXTENSION_PACKAGES_S3_PREFIX": "production", - "EXTENSION_ASSETS_S3_ENDPOINT": "https://s3.eu-central-003.backblazeb2.com", - "EXTENSION_ASSETS_S3_REGION": "eu-central-003", - "EXTENSION_ASSETS_S3_BUCKET": "fb-extensions-assets", "EXTENSION_ASSETS_PUBLIC_URL": "https://extensions-assets.fossbilling.net", "EXTENSIONS_RETENTION_MODE": "dry-run", "EXTENSIONS_V2_EMAIL_PROVIDER": "mxroute", - "EXTENSIONS_V2_MXROUTE_SERVER": "wednesday.mxrouting.net", - "EXTENSIONS_V2_MXROUTE_USERNAME": "extensions@fossbilling.org", "M2M_ISSUER": "https://auth.fossbilling.net", "M2M_JWKS_URL": "https://auth.fossbilling.net/.well-known/jwks.json", "M2M_AUDIENCE": "https://api.fossbilling.net" From b97b114f9b93e52baab918449f1355915d0ad284 Mon Sep 17 00:00:00 2001 From: Adam Daley Date: Tue, 10 Nov 2026 02:42:22 +0000 Subject: [PATCH 10/12] Consolidate package runtime guidance --- src/services/extensions/v2/PACKAGE-RUNTIME.md | 38 ------------------- src/services/extensions/v2/PACKAGING.md | 7 ++-- 2 files changed, 4 insertions(+), 41 deletions(-) delete mode 100644 src/services/extensions/v2/PACKAGE-RUNTIME.md diff --git a/src/services/extensions/v2/PACKAGE-RUNTIME.md b/src/services/extensions/v2/PACKAGE-RUNTIME.md deleted file mode 100644 index 3d13bd59..00000000 --- a/src/services/extensions/v2/PACKAGE-RUNTIME.md +++ /dev/null @@ -1,38 +0,0 @@ -# Package runtime validation - -Package preparation accepts ready-to-install ZIPs. The directory never runs -uploaded code, compilers, dependency installation or package build scripts. - -The HTTP endpoint validates the bounded request, stores the upload privately and -returns a job ID. The queue consumer validates ZIP entries, copies their existing -compressed payloads, replaces the manifest and prepares the display icon. - -## Local checks - -Run `npm run test:all`. Worker tests cover the API routes, guarded D1 writes, -archive limits and corruption, ownership changes, publisher revocation, private -previews, approval and cleanup. Storage and image bindings are emulated for these -tests. Node tests exercise package preparation and job processing through their -interfaces, plus the B2 adapter's request and integrity checks. - -The directory tests cover the private job proxy and polling failures; publisher -tests cover completion, rejection and the polling deadline. - -## Real B2 smoke test - -A local Worker smoke test against both real B2 buckets on 7 November 2026 -verified pending SVG/AVIF/PNG CDN delivery, checksums, security headers and cache -hits. Approval and manifest corrections reused existing images. Cleanup removed -abandoned object versions while preserving referenced objects. Test objects and -CDN cache entries were removed afterward. No Worker deployment or production -database changes were made. - -## Before production enablement - -Measure admission and consumer invocations separately: CPU and wall duration, -memory failures, queue delay, retries, and storage/image-service latency. Include -near-limit ZIPs and SVG/raster icons. The consumer is configured for one message -at a time with concurrency one. - -Local functional tests and earlier process-CPU benchmarks do not establish -production capacity or billing. diff --git a/src/services/extensions/v2/PACKAGING.md b/src/services/extensions/v2/PACKAGING.md index 31abf6e9..f00f24e0 100644 --- a/src/services/extensions/v2/PACKAGING.md +++ b/src/services/extensions/v2/PACKAGING.md @@ -279,6 +279,7 @@ The disabled mode blocks package reads as well as writes. Once managed releases are published, disabling it also stops their API downloads and private previews; it is not an upload-only pause switch. Prepared CDN images remain accessible independently, even for pending packages. -Local validation and the remaining production measurements are summarized in -[PACKAGE-RUNTIME.md](PACKAGE-RUNTIME.md). Local queue success does not establish -production CPU, memory or latency limits. +Before enabling package mode, measure admission and queue-consumer invocations +separately: CPU and wall duration, memory failures, queue delay, retries, and +storage/image-service latency. Include near-limit ZIPs and SVG/raster icons. +Local functional tests do not establish production capacity or billing. From 63d99d4d5618aed7168de25205a9025bc216136a Mon Sep 17 00:00:00 2001 From: Adam Daley Date: Tue, 10 Nov 2026 02:53:27 +0000 Subject: [PATCH 11/12] Remove the package R2 fallback and test through B2 --- src/lib/adapters/cloudflare/package-jobs.ts | 7 +- src/lib/adapters/cloudflare/package-store.ts | 73 +---- src/services/extensions/v2/PACKAGING.md | 19 +- test/env.d.ts | 1 - test/services/extensions/v2/packages.test.ts | 284 ++++++++++--------- vitest.config.ts | 1 - 6 files changed, 171 insertions(+), 214 deletions(-) diff --git a/src/lib/adapters/cloudflare/package-jobs.ts b/src/lib/adapters/cloudflare/package-jobs.ts index 70a1cee4..8e50757e 100644 --- a/src/lib/adapters/cloudflare/package-jobs.ts +++ b/src/lib/adapters/cloudflare/package-jobs.ts @@ -14,10 +14,9 @@ import { PackageError } from "../../../services/extensions/v2/packages/errors"; import { configuredPackageStore } from "./package-store"; import { CloudflarePackageImages } from "./package-images"; -type PackageBindings = CloudflareBindings & { - EXTENSION_PACKAGE_BUCKET?: R2Bucket; -}; -export function packagePreparation(env: PackageBindings): PackagePreparation { +export function packagePreparation( + env: CloudflareBindings +): PackagePreparation { const bindings = createCloudflareBindings(env); const getEnv = (name: string) => bindings.environment.get(name); const store = configuredPackageStore(env, getEnv); diff --git a/src/lib/adapters/cloudflare/package-store.ts b/src/lib/adapters/cloudflare/package-store.ts index 5bcf5aa7..cfc073a8 100644 --- a/src/lib/adapters/cloudflare/package-store.ts +++ b/src/lib/adapters/cloudflare/package-store.ts @@ -11,7 +11,7 @@ export function packageStore( } export function configuredPackageStore( - env: CloudflareBindings & { EXTENSION_PACKAGE_BUCKET?: R2Bucket }, + env: CloudflareBindings, getEnv: (name: string) => string | undefined ): PackageStore | null { const mode = getEnv("EXTENSION_PACKAGES_MODE"); @@ -50,72 +50,13 @@ export function configuredPackageStore( ); } } - const packages = getEnv("EXTENSION_PACKAGES_S3_ENDPOINT") - ? b2ObjectStore(getEnv, "packages") - : null; - const assets = getEnv("EXTENSION_ASSETS_S3_ENDPOINT") - ? b2ObjectStore(getEnv, "assets") - : null; - const bucket = env.EXTENSION_PACKAGE_BUCKET; - if (!packages && !bucket) return null; - const localBucket = () => { - if (!bucket) - throw new PackageError( - "PACKAGES_DISABLED", - "Storage is not configured", - 503 - ); - return bucket; - }; - const remoteFor = (key: string) => + const packages = b2ObjectStore(getEnv, "packages"); + const assets = b2ObjectStore(getEnv, "assets"); + const storeFor = (key: string) => key.startsWith("extensions/") ? assets : packages; return { - async put(key, bytes, checksum) { - const remote = remoteFor(key); - if (remote) return remote.put(key, bytes, checksum); - const result = await localBucket().put(key, bytes, { - onlyIf: { etagDoesNotMatch: "*" }, - sha256: checksum, - httpMetadata: { - contentType: key.endsWith(".avif") - ? "image/avif" - : key.endsWith(".png") - ? "image/png" - : key.endsWith(".svg") - ? "image/svg+xml" - : "application/zip", - cacheControl: "private, no-store" - } - }); - if (!result) throw new Error("Package object already exists"); - }, - async get(key, checksum) { - const remote = remoteFor(key); - if (remote) return remote.get(key, checksum); - const object = await localBucket().get(key); - if (!object) return null; - if (checksum) { - const actual = object.checksums.sha256; - const hex = actual - ? Array.from(new Uint8Array(actual), (byte) => - byte.toString(16).padStart(2, "0") - ).join("") - : null; - if (hex !== checksum) { - await object.body.cancel(); - throw new PackageError( - "PACKAGE_UNAVAILABLE", - "Stored object failed checksum verification", - 503 - ); - } - } - return object.body; - }, - async delete(key) { - const remote = remoteFor(key); - if (remote) return remote.delete(key); - await localBucket().delete(key); - } + put: (key, bytes, checksum) => storeFor(key).put(key, bytes, checksum), + get: (key, checksum) => storeFor(key).get(key, checksum), + delete: (key) => storeFor(key).delete(key) }; } diff --git a/src/services/extensions/v2/PACKAGING.md b/src/services/extensions/v2/PACKAGING.md index f00f24e0..7cabbd67 100644 --- a/src/services/extensions/v2/PACKAGING.md +++ b/src/services/extensions/v2/PACKAGING.md @@ -5,7 +5,7 @@ complete. The API validates and repacks them with directory-generated manifests, then publishes them through the existing revision review. It never runs uploaded code or package build scripts. ZIPs use the private B2 bucket; prepared icons use the public asset bucket, including icons for pending submissions. Automated tests -inject emulated storage. +use the B2 adapter with mocked S3 responses. Package routes are disabled by default. Local development and production enablement are separate modes. ## Local development and testing @@ -18,8 +18,8 @@ and Images bindings run locally under Wrangler; B2 requests use the configured buckets and development keys, so use a separate key and package object prefix. `npm run test:all` covers the package lifecycle through the real API routes and -D1 repository, with emulated storage and images. Node tests also exercise package -preparation and B2 storage without a Cloudflare runtime. +D1 repository, with mocked S3 responses and emulated image processing. Node tests +also exercise package preparation and B2 storage without a Cloudflare runtime. ## API and review @@ -167,13 +167,12 @@ EXTENSION_PACKAGES_S3_PREFIX="local" Keep this bucket private. Use a separate development key restricted to the bucket with read, write and delete-file permissions. No public URL or domain is required. Use `EXTENSION_PACKAGES_MODE=local` only outside production. The committed production configuration remains disabled. -The tests inject a local R2 store when S3 endpoints are unset. Normal local development -uses B2 with development credentials. An incomplete S3 configuration or upstream +Tests run the B2 adapter against mocked S3 responses. Local development uses B2 +with development credentials. An incomplete S3 configuration or upstream failure returns 503 rather than silently switching storage backends. -R2 validates the supplied SHA-256 when writing and returns that checksum as -object metadata. Its adapter checks this metadata on reads without rehashing the -response body. The B2 adapter additionally hashes the downloaded bytes. +The B2 adapter signs uploads with their SHA-256 payload, stores checksum metadata +and verifies both metadata and downloaded bytes on reads. Source and built ZIPs use `local/packages/`. Object IDs are allocated once by D1 before upload. Uploads carry a signed SHA-256 payload and private cache @@ -201,7 +200,7 @@ EXTENSION_ASSETS_PUBLIC_URL="https://extensions-assets.fossbilling.net" Restrict the asset key to this bucket with read, write and delete-file permissions. Pending API previews read these stored objects; cleanup deletes abandoned versions. -Tests can inject emulated storage when the endpoint is unset. Public object keys use +Tests mock S3 responses without accessing a real bucket. Public object keys use `extensions//icons/.`, for example: ```text @@ -251,7 +250,7 @@ or secrets are created by a local build or type-generation command. Before deployment (including an automatic deployment after merge): -1. Provision the `extension-packages` Queue. No R2 image bucket is needed; +1. Provision the `extension-packages` Queue. ZIPs use private B2 and prepared icons use the public B2 asset bucket. The queue binding must exist even while package mode is disabled; that flag blocks application routes, not deployment-time binding validation. diff --git a/test/env.d.ts b/test/env.d.ts index 8d6cd677..48c5a596 100644 --- a/test/env.d.ts +++ b/test/env.d.ts @@ -11,7 +11,6 @@ declare module "cloudflare:test" { // hence augmenting this namespace directly rather than the module above. declare namespace Cloudflare { interface Env { - EXTENSION_PACKAGE_BUCKET: R2Bucket; TEST_MIGRATIONS_EXTENSIONS: import("cloudflare:test").D1Migration[]; TEST_MIGRATIONS_CENTRAL_ALERTS: import("cloudflare:test").D1Migration[]; } diff --git a/test/services/extensions/v2/packages.test.ts b/test/services/extensions/v2/packages.test.ts index 6db82c2f..25d525e3 100644 --- a/test/services/extensions/v2/packages.test.ts +++ b/test/services/extensions/v2/packages.test.ts @@ -55,14 +55,71 @@ const manifest = () => "package-demo", ExtensionContentSchema.parse(sampleContent()) ); -const bucket = () => env.EXTENSION_PACKAGE_BUCKET!; +const packageKey = (id: string) => `/test-packages/local/packages/${id}`; +let b2: ReturnType; + +function mockB2() { + Object.assign(env, { + EXTENSION_PACKAGES_S3_ENDPOINT: "https://s3.example.test", + EXTENSION_PACKAGES_S3_REGION: "test-region", + EXTENSION_PACKAGES_S3_BUCKET: "test-packages", + EXTENSION_PACKAGES_S3_PREFIX: "local", + EXTENSION_PACKAGES_S3_ACCESS_KEY_ID: "test-key", + EXTENSION_PACKAGES_S3_SECRET_ACCESS_KEY: "test-secret", + EXTENSION_ASSETS_S3_ENDPOINT: "https://s3.example.test", + EXTENSION_ASSETS_S3_REGION: "test-region", + EXTENSION_ASSETS_S3_BUCKET: "test-assets", + EXTENSION_ASSETS_S3_ACCESS_KEY_ID: "test-key", + EXTENSION_ASSETS_S3_SECRET_ACCESS_KEY: "test-secret", + EXTENSION_ASSETS_PUBLIC_URL: "https://extensions-assets.fossbilling.net" + }); + const calls: Request[] = []; + const objects = new Map< + string, + { + bytes: ArrayBuffer; + checksum: string; + version: string; + } + >(); + const fetch = vi.fn(async (request: Request) => { + const url = new URL(request.url); + expect(url.hostname).toBe("s3.example.test"); + expect(request.headers.get("Authorization")).toMatch(/^AWS4-HMAC-SHA256 /); + calls.push(request.clone()); + const key = url.pathname; + if (request.method === "PUT") { + objects.set(key, { + bytes: await request.arrayBuffer(), + checksum: request.headers.get("x-amz-meta-sha256")!, + version: crypto.randomUUID() + }); + return new Response(null); + } + const object = objects.get(key); + if (!object) return new Response(null, { status: 404 }); + if (request.method === "DELETE") { + expect(url.searchParams.get("versionId")).toBe(object.version); + objects.delete(key); + return new Response(null, { status: 204 }); + } + expect(["GET", "HEAD"]).toContain(request.method); + return new Response(request.method === "HEAD" ? null : object.bytes, { + headers: { + "x-amz-meta-sha256": object.checksum, + "x-amz-version-id": object.version + } + }); + }); + vi.stubGlobal("fetch", fetch); + return { calls, objects, fetch }; +} beforeEach(async () => { vi.spyOn(env.EXTENSION_PACKAGE_QUEUE, "send").mockResolvedValue({ metadata: { metrics: { backlogCount: 0, backlogBytes: 0 } } }); - Reflect.deleteProperty(env, "EXTENSION_PACKAGES_S3_ENDPOINT"); - Reflect.deleteProperty(env, "EXTENSION_ASSETS_S3_ENDPOINT"); + b2 = mockB2(); Object.assign(env, { ENVIRONMENT: "local", EXTENSION_PACKAGES_MODE: "local", @@ -220,6 +277,14 @@ async function unpack(bytes: Uint8Array) { describe("publisher package builds", () => { beforeEach(async () => { await activateM2mJwks(); + const jwksFetch = globalThis.fetch; + vi.stubGlobal("fetch", (input: RequestInfo | URL, init?: RequestInit) => { + const request = + input instanceof Request ? input : new Request(input, init); + return new URL(request.url).hostname === "s3.example.test" + ? b2.fetch(request) + : jwksFetch(input, init); + }); await insertUser(db, { id: "publisher-moderator", is_moderator: 1 }); const response = await put( `${root}/m2m/clients/package-publisher`, @@ -362,28 +427,20 @@ describe("publisher package builds", () => { expect((await machineBuild()).status).toBe(403); }); it("rechecks revocation after uploading before linking source bytes", async () => { - const original = bucket().put.bind(bucket()); - const write = vi - .spyOn(bucket(), "put") - .mockImplementationOnce(async (...args) => { - const result = await original(...args); - await db - .prepare( - "UPDATE m2m_publisher_clients SET revoked_at=CURRENT_TIMESTAMP WHERE client_id='package-publisher'" - ) - .run(); - return result; - }); - try { - expect((await machineBuild()).status).toBe(403); - expect( - await db - .prepare("SELECT count(*) as count FROM package_sources") - .first() - ).toEqual({ count: 0 }); - } finally { - write.mockRestore(); - } + const original = b2.fetch.getMockImplementation()!; + b2.fetch.mockImplementationOnce(async (request) => { + const result = await original(request); + await db + .prepare( + "UPDATE m2m_publisher_clients SET revoked_at=CURRENT_TIMESTAMP WHERE client_id='package-publisher'" + ) + .run(); + return result; + }); + expect((await machineBuild()).status).toBe(403); + expect( + await db.prepare("SELECT count(*) as count FROM package_sources").first() + ).toEqual({ count: 0 }); }); }); @@ -523,7 +580,9 @@ describe("queued package preparation", () => { const { result } = (await response.json()) as { result: { id: string } }; const jobs = new PackageJobsDatabase(db); const job = (await jobs.get(result.id))!; - await bucket().delete(job.uploadObjectId!); + const key = packageKey(job.uploadObjectId!); + const staged = b2.objects.get(key)!; + b2.objects.delete(key); expect( await processPackageJob(jobs, job.id, (job) => prepareQueuedPackage(packagePreparation(env), job) @@ -534,9 +593,7 @@ describe("queued package preparation", () => { attempts: 1, error: { code: "SOURCE_UNAVAILABLE", status: 503 } }); - await bucket().put(job.uploadObjectId!, source, { - sha256: job.uploadSha256! - }); + b2.objects.set(key, staged); expect( await processPackageJob(jobs, job.id, (job) => prepareQueuedPackage(packagePreparation(env), job) @@ -612,14 +669,7 @@ describe("queued package preparation", () => { describe("stored directory packages", () => { it("uses separate B2 buckets for ZIPs and icons through preparation, correction and approval", async () => { - const remote = configureB2({ - EXTENSION_PACKAGES_S3_ENDPOINT: "https://s3.example.test", - EXTENSION_PACKAGES_S3_REGION: "eu-central-003", - EXTENSION_PACKAGES_S3_BUCKET: "test-packages", - EXTENSION_PACKAGES_S3_ACCESS_KEY_ID: "test-key", - EXTENSION_PACKAGES_S3_SECRET_ACCESS_KEY: "test-secret" - }); - const requests = remote.calls; + const requests = b2.calls; const writer = new ZipWriter(new Uint8ArrayWriter(), options); await writer.add("Service.php", new TextReader(" { .prepare("SELECT object_id FROM package_artifacts WHERE id=?") .bind(artifact.id) .first<{ object_id: string }>(); - await bucket().delete(object!.object_id); + b2.objects.delete(packageKey(object!.object_id)); // Reading the icon must not depend on opening the ZIP. expect((await get(path, await authHeaders("owner"))).status).toBe(200); expect((await approve(await submitted(artifact))).status).toBe(200); expect((await get(path, {})).status).toBe(200); const published = await getExtension(db, "package-demo"); expect(published?.icon_url).toBe( - artifact.download_url.replace(/\/download$/, "/icon") + `https://extensions-assets.fossbilling.net/${String(before!.svg_object_id)}` ); await db .prepare( @@ -971,50 +1021,6 @@ describe("stored directory packages", () => { expect((await upload()).status).toBe(200); }); - function configureB2(overrides: Record = {}) { - Object.assign(env, { - EXTENSION_ASSETS_S3_ENDPOINT: "https://s3.example.test", - EXTENSION_ASSETS_S3_REGION: "eu-central-003", - EXTENSION_ASSETS_S3_BUCKET: "test-assets", - EXTENSION_ASSETS_S3_ACCESS_KEY_ID: "test-key", - EXTENSION_ASSETS_S3_SECRET_ACCESS_KEY: "test-secret", - EXTENSION_ASSETS_PUBLIC_URL: "https://extensions-assets.fossbilling.net", - ...overrides - }); - const calls: Request[] = []; - const objects = new Map(); - let failure = false; - vi.stubGlobal( - "fetch", - async (input: RequestInfo | URL, init?: RequestInit) => { - const request = - input instanceof Request ? input : new Request(input, init); - expect(new URL(request.url).hostname).toBe("s3.example.test"); - calls.push(request.clone()); - if (failure) return new Response(null, { status: 503 }); - const key = new URL(request.url).pathname; - if (request.method === "PUT") { - objects.set(key, { - bytes: await request.arrayBuffer(), - checksum: request.headers.get("x-amz-meta-sha256")! - }); - return new Response(null); - } - const object = objects.get(key); - if (!object) return new Response(null, { status: 404 }); - expect(request.method).toBe("GET"); - return new Response(object.bytes, { - headers: { "x-amz-meta-sha256": object.checksum } - }); - } - ); - return { - calls, - fail: (value: boolean) => { - failure = value; - } - }; - } async function iconSource(format: "svg" | "png" = "svg") { const svg = ''; @@ -1038,7 +1044,6 @@ describe("stored directory packages", () => { return writer.close(); } it("stores pending icons at unlisted B2 URLs using the permanent draft UUID", async () => { - const remote = configureB2(); const artifact = await built(manifest(), await iconSource()); const draft = (await getExtension(db, "package-demo"))!; expect(draft.id).toMatch(UUID_V7); @@ -1048,14 +1053,18 @@ describe("stored directory packages", () => { .status ).toBe(200); expect((await get(`${root}/extensions/package-demo`, {})).status).toBe(404); - const uploads = remote.calls.filter((r) => r.method === "PUT"); + const uploads = b2.calls.filter( + (r) => + r.method === "PUT" && + new URL(r.url).pathname.startsWith("/test-assets/") + ); expect(uploads).toHaveLength(1); const upload = uploads[0]; const objectKey = new URL(upload.url).pathname.replace("/test-assets/", ""); expect(objectKey).toMatch( new RegExp(`^extensions/${draft.id}/icons/[a-f0-9-]{36}\\.svg$`) ); - const callsBefore = remote.calls.length; + const callsBefore = b2.calls.length; expect((await get(`${root}/packages/${artifact.id}/icon`, {})).status).toBe( 404 ); @@ -1067,7 +1076,7 @@ describe("stored directory packages", () => { ) ).status ).toBe(404); - expect(remote.calls).toHaveLength(callsBefore); + expect(b2.calls).toHaveLength(callsBefore); const preview = await get( `${root}/packages/${artifact.id}/icon`, await authHeaders("owner") @@ -1075,16 +1084,16 @@ describe("stored directory packages", () => { expect(preview.status).toBe(200); expect(preview.headers.get("Cache-Control")).toBe("private, no-store"); expect(await preview.text()).toContain(" { expect((await getExtension(db, "package-demo"))?.icon_url).toBe( published.icon_url ); - expect(remote.calls.filter((r) => r.method === "PUT")).toHaveLength(1); + expect( + b2.calls.filter( + (r) => + r.method === "PUT" && + new URL(r.url).pathname.startsWith("/test-assets/") + ) + ).toHaveLength(1); }); it("fails preparation on asset upload errors and retries without changing storage backends", async () => { - const remote = configureB2(); const source = await iconSource(); - remote.fail(true); + const original = b2.fetch.getMockImplementation()!; + b2.fetch.mockImplementation(async (request) => + request.method === "PUT" && + new URL(request.url).pathname.startsWith("/test-assets/") + ? new Response(null, { status: 503 }) + : original(request) + ); expect((await upload(manifest(), source)).status).toBe(503); expect( await db .prepare("SELECT count(*) AS count FROM package_artifacts") .first() ).toEqual({ count: 0 }); - remote.fail(false); + b2.fetch.mockImplementation(original); const artifact = await built(manifest(), source); expect( ( @@ -1131,7 +1151,6 @@ describe("stored directory packages", () => { ); }); it("publishes an icon when a valid extension ID is supplied in uppercase", async () => { - configureB2(); const artifact = await built(manifest(), await iconSource()); const revision = await submitted(artifact); await insertUser(db, { id: "moderator", is_moderator: 1 }); @@ -1149,10 +1168,13 @@ describe("stored directory packages", () => { ); }); it("publishes AVIF and PNG with one immutable image identity for browser fallback", async () => { - const remote = configureB2(); const artifact = await built(manifest(), await iconSource("png")); expect((await approve(await submitted(artifact))).status).toBe(200); - const uploads = remote.calls.filter((request) => request.method === "PUT"); + const uploads = b2.calls.filter( + (request) => + request.method === "PUT" && + new URL(request.url).pathname.startsWith("/test-assets/") + ); expect(uploads).toHaveLength(2); const extension = (await getExtension(db, "package-demo"))!; expect(new URL(uploads[0].url).pathname).toMatch( @@ -1209,7 +1231,7 @@ describe("stored directory packages", () => { .prepare("SELECT object_id FROM package_sources WHERE id=?") .bind(artifact.source_id) .first<{ object_id: string }>(); - await bucket().delete(source!.object_id); + b2.objects.delete(packageKey(source!.object_id)); expect((await built(manifest(), undefined, artifact.source_id)).id).toBe( artifact.id ); @@ -1447,7 +1469,9 @@ describe("stored directory packages", () => { .prepare("SELECT object_id FROM package_sources WHERE id=?") .bind(artifact.source_id) .first<{ object_id: string }>(); - await bucket().put(source!.object_id, "corrupt"); + b2.objects.get(packageKey(source!.object_id))!.bytes = new Uint8Array( + new TextEncoder().encode("corrupt") + ).buffer; expect( ( await upload( @@ -1461,7 +1485,9 @@ describe("stored directory packages", () => { .prepare("SELECT object_id FROM package_artifacts WHERE id=?") .bind(artifact.id) .first<{ object_id: string }>(); - await bucket().put(stored!.object_id, "corrupt"); + b2.objects.get(packageKey(stored!.object_id))!.bytes = new Uint8Array( + new TextEncoder().encode("corrupt") + ).buffer; expect( ( await get( @@ -1571,28 +1597,14 @@ describe("stored directory packages", () => { expect((await upload()).status).toBe(503); // localhost URL / incomplete storage env.EXTENSION_PACKAGES_BASE_URL = "https://api.fossbilling.net/extensions/v2"; - expect((await upload()).status).toBe(503); // must not fall back to local R2 - Object.assign(env, { - EXTENSION_PACKAGES_S3_ENDPOINT: "https://s3.example.test", - EXTENSION_PACKAGES_S3_REGION: "eu-central-003", - EXTENSION_PACKAGES_S3_BUCKET: "private-packages", - EXTENSION_PACKAGES_S3_PREFIX: "production", - EXTENSION_PACKAGES_S3_ACCESS_KEY_ID: "test-key", - EXTENSION_PACKAGES_S3_SECRET_ACCESS_KEY: "test-secret" - }); + expect((await upload()).status).toBe(503); // local object prefix cannot enable production + Object.assign(env, { EXTENSION_PACKAGES_S3_PREFIX: "production" }); + Reflect.deleteProperty(env, "EXTENSION_ASSETS_S3_SECRET_ACCESS_KEY"); + expect((await upload()).status).toBe(503); Object.assign(env, { - EXTENSION_ASSETS_S3_ENDPOINT: "https://s3.example.test", - EXTENSION_ASSETS_S3_REGION: "eu-central-003", - EXTENSION_ASSETS_S3_BUCKET: "public-assets", - EXTENSION_ASSETS_S3_ACCESS_KEY_ID: "test-key", - EXTENSION_ASSETS_S3_SECRET_ACCESS_KEY: "test-secret", - EXTENSION_ASSETS_PUBLIC_URL: "https://extensions-assets.fossbilling.net" - }); - const requests: Request[] = []; - vi.stubGlobal("fetch", async (request: Request) => { - requests.push(request); - return new Response(null, { status: 200 }); + EXTENSION_ASSETS_S3_SECRET_ACCESS_KEY: "test-secret" }); + const requests = b2.calls; const response = await upload( manifest(), undefined, @@ -1621,6 +1633,11 @@ describe("stored directory packages", () => { expect( (await upload(manifest(), new Uint8Array(5 * 1024 * 1024 + 1))).status ).toBe(413); + Reflect.deleteProperty(env, "EXTENSION_PACKAGES_S3_ENDPOINT"); + expect((await upload()).status).toBe(503); + Object.assign(env, { + EXTENSION_PACKAGES_S3_ENDPOINT: "https://s3.example.test" + }); await db .prepare("UPDATE users SET deleted_at=CURRENT_TIMESTAMP WHERE id='owner'") .run(); @@ -1630,14 +1647,8 @@ describe("stored directory packages", () => { }); it("leaves failed uploads reserved for cleanup and permits a safe retry", async () => { - const failure = vi - .spyOn(bucket(), "put") - .mockRejectedValueOnce(new Error("storage outage")); - try { - expect((await upload()).status).toBe(503); - } finally { - failure.mockRestore(); - } + b2.fetch.mockResolvedValueOnce(new Response(null, { status: 503 })); + expect((await upload()).status).toBe(503); expect( await db .prepare("SELECT count(*) AS count FROM package_artifacts") @@ -1654,13 +1665,19 @@ describe("stored directory packages", () => { it("counts abandoned reservations against quota and safely cleans them after a day", async () => { const artifact = await built(); + const abandoned = crypto.randomUUID(); await db .prepare( - "INSERT INTO package_objects (id,created_by,bytes,created_at) VALUES ('abandoned','owner',?,datetime('now','-2 days'))" + "INSERT INTO package_objects (id,created_by,bytes,created_at) VALUES (?,'owner',?,datetime('now','-2 days'))" ) - .bind(512 * 1024 * 1024) + .bind(abandoned, 512 * 1024 * 1024) .run(); - await bucket().put("abandoned", "failed upload"); + const bytes = new Uint8Array(new TextEncoder().encode("failed upload")); + b2.objects.set(packageKey(abandoned), { + bytes: bytes.buffer, + checksum: await sha256(bytes), + version: "abandoned-version" + }); expect( ( await upload( @@ -1677,7 +1694,10 @@ describe("stored directory packages", () => { ); expect(cleanup.status).toBe(200); expect(await cleanup.json()).toEqual({ result: { deleted: 1 } }); - expect(await bucket().get("abandoned")).toBeNull(); + expect(b2.objects.has(packageKey(abandoned))).toBe(false); + expect( + b2.calls.find((request) => request.method === "DELETE")?.url + ).toContain("versionId=abandoned-version"); expect( ( await upload( diff --git a/vitest.config.ts b/vitest.config.ts index bdf5057d..0fd362d1 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -18,7 +18,6 @@ export default defineConfig({ cloudflareTest({ wrangler: { configPath: "./wrangler.jsonc" }, miniflare: { - r2Buckets: ["EXTENSION_PACKAGE_BUCKET"], images: { binding: "PACKAGE_IMAGES" }, // Add test environment variables bindings: { From 53c416c7f1b54e284cc74347044c264c51febc77 Mon Sep 17 00:00:00 2001 From: Adam Daley Date: Tue, 10 Nov 2026 03:14:34 +0000 Subject: [PATCH 12/12] Remove unused upload fallback and fix S3 test assertions --- src/services/extensions/v2/packages/build.ts | 36 ++++++------------- .../extensions/v2/packages/prepare.ts | 10 +++--- test/services/extensions/v2/packages.test.ts | 18 +++++++--- 3 files changed, 31 insertions(+), 33 deletions(-) diff --git a/src/services/extensions/v2/packages/build.ts b/src/services/extensions/v2/packages/build.ts index 3a6f3a2c..371aef56 100644 --- a/src/services/extensions/v2/packages/build.ts +++ b/src/services/extensions/v2/packages/build.ts @@ -41,8 +41,7 @@ export async function buildStoredPackage(input: { store: PackageStore; owner: PackageOwner; manifest: Manifest; - source?: Uint8Array; - sourceObjectId?: string; + upload?: { bytes: Uint8Array; objectId: string }; sourceId?: string; baseUrl: string; images: PackageImageProcessor; @@ -59,7 +58,7 @@ export async function buildStoredPackage(input: { "Source not found for this extension version", 404 ); - let sourceBytes = input.source; + let sourceBytes = input.upload?.bytes; if (sourceBytes && source && (await sha256(sourceBytes)) !== source.sha256) throw new PackageError( "SOURCE_CHANGED", @@ -104,16 +103,8 @@ export async function buildStoredPackage(input: { ? [] : await input.images.prepare(built.icon, built.iconName === "icon.svg"); if (!source) { - const objectId = - input.sourceObjectId ?? - (await reserveObject( - repository, - store, - owner, - manifest.id, - sourceBytes, - built.sourceSha256 - )); + // Without an upload, the stored-source lookup above must have succeeded. + const objectId = input.upload!.objectId; source = await repository.linkSource(owner, manifest, { id: crypto.randomUUID(), sha256: built.sourceSha256, @@ -125,19 +116,14 @@ export async function buildStoredPackage(input: { "Ownership changed while building", 403 ); - if (source.object_id !== objectId) { - if (!input.sourceObjectId) { - await store.delete(objectId); - await repository.removeObject(objectId); - } - if (source.sha256 !== built.sourceSha256) - throw new PackageError( - "SOURCE_CHANGED", - "Another upload reserved this version with different source files", - 409 - ); - } + if (source.sha256 !== built.sourceSha256) + throw new PackageError( + "SOURCE_CHANGED", + "Another upload reserved this version with different source files", + 409 + ); } + const extensionId = await repository.ensureDraft(owner, manifest.id); await storePackageIcon({ repository, diff --git a/src/services/extensions/v2/packages/prepare.ts b/src/services/extensions/v2/packages/prepare.ts index 42bb791c..35061266 100644 --- a/src/services/extensions/v2/packages/prepare.ts +++ b/src/services/extensions/v2/packages/prepare.ts @@ -41,7 +41,7 @@ export async function prepareQueuedPackage( "Package ownership changed while queued", 403 ); - let source: Uint8Array | undefined; + let upload: { bytes: Uint8Array; objectId: string } | undefined; if (job.uploadObjectId) { const body = await store.get( job.uploadObjectId, @@ -53,15 +53,17 @@ export async function prepareQueuedPackage( "Queued source ZIP is unavailable", 503 ); - source = await boundedBytes(body, PACKAGE_LIMITS.sourceBytes); + upload = { + bytes: await boundedBytes(body, PACKAGE_LIMITS.sourceBytes), + objectId: job.uploadObjectId + }; } const artifact = await buildStoredPackage({ repository, store, owner: current, manifest: job.manifest, - source, - sourceObjectId: job.uploadObjectId ?? undefined, + upload, sourceId: job.sourceId ?? undefined, baseUrl: services.baseUrl.replace(/\/$/, ""), images: services.images diff --git a/test/services/extensions/v2/packages.test.ts b/test/services/extensions/v2/packages.test.ts index 25d525e3..8db2efeb 100644 --- a/test/services/extensions/v2/packages.test.ts +++ b/test/services/extensions/v2/packages.test.ts @@ -84,8 +84,6 @@ function mockB2() { >(); const fetch = vi.fn(async (request: Request) => { const url = new URL(request.url); - expect(url.hostname).toBe("s3.example.test"); - expect(request.headers.get("Authorization")).toMatch(/^AWS4-HMAC-SHA256 /); calls.push(request.clone()); const key = url.pathname; if (request.method === "PUT") { @@ -99,11 +97,13 @@ function mockB2() { const object = objects.get(key); if (!object) return new Response(null, { status: 404 }); if (request.method === "DELETE") { - expect(url.searchParams.get("versionId")).toBe(object.version); + if (url.searchParams.get("versionId") !== object.version) + return new Response(null, { status: 400 }); objects.delete(key); return new Response(null, { status: 204 }); } - expect(["GET", "HEAD"]).toContain(request.method); + if (!["GET", "HEAD"].includes(request.method)) + return new Response(null, { status: 405 }); return new Response(request.method === "HEAD" ? null : object.bytes, { headers: { "x-amz-meta-sha256": object.checksum, @@ -129,6 +129,7 @@ beforeEach(async () => { await seedDeveloper("developer", "owner"); }); afterEach(() => { + const requests = b2.fetch.mock.calls.map(([request]) => request); vi.restoreAllMocks(); vi.unstubAllGlobals(); for (const key of [ @@ -150,6 +151,15 @@ afterEach(() => { ]) Reflect.deleteProperty(env, `EXTENSION_PACKAGES_S3_${suffix}`); env.EXTENSION_PACKAGES_MODE = "disabled"; + // Assert outside the adapter's catch, including calls with injected failures. + for (const request of requests) { + const url = new URL(request.url); + expect(url.hostname).toBe("s3.example.test"); + expect(request.headers.get("Authorization")).toMatch(/^AWS4-HMAC-SHA256 /); + expect(["PUT", "GET", "HEAD", "DELETE"]).toContain(request.method); + if (request.method === "DELETE") + expect(url.searchParams.get("versionId")).toBeTruthy(); + } }); async function zip(content = "