diff --git a/.gitignore b/.gitignore index 1b94e00d..481f31b5 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ node_modules/ .env .env.production .dev.vars +.dev.vars.* # logs logs/ diff --git a/README.md b/README.md index 8019d258..0f4186fb 100644 --- a/README.md +++ b/README.md @@ -76,6 +76,17 @@ We use [Cloudflare D1](https://developers.cloudflare.com/d1/) and [KV](https://d ### Environment Variables +Production B2 connection details and MXroute server/username are managed in the +Worker dashboard under **Settings > Variables and Secrets** as runtime variables. +Credentials are stored as secrets. Build environment variables do not configure +the running Worker. `keep_vars: true` preserves dashboard-managed variables on +Wrangler deployments; keep their values out of the committed `vars` map and docs. +Public URLs and deployment controls remain in `wrangler.jsonc`. + +For local development, put operational values and credentials in the ignored +`.dev.vars` file. Document variable names and placeholders, not production values. +Package storage configuration is listed in [PACKAGING.md](src/services/extensions/v2/PACKAGING.md). + - `GITHUB_TOKEN`: A GitHub Personal Access Token (classic) with public repo read access. - `ASSERTION_SIGNING_SECRET`: Shared HMAC secret used to verify the short-lived bearer assertions minted by the Extensions site. Configure the same value in both Workers; it is never sent to clients. - `ASSERTION_SIGNING_SECRET_PREVIOUS`: Optional previous HMAC secret accepted during a signing-key rotation. @@ -83,7 +94,8 @@ We use [Cloudflare D1](https://developers.cloudflare.com/d1/) and [KV](https://d - `EXTENSIONS_V2_EMAIL_PROVIDER`: `mxroute` or `resend` to send moderator notification emails, or unset/`disabled` to skip sending. - `EXTENSIONS_V2_EMAIL_FROM`: Sender address for notification emails (default `noreply@fossbilling.org`). - `EXTENSIONS_V2_EMAIL_REPLY_TO`: Optional Reply-To address for a monitored inbox. Omit when unset (no Reply-To header is sent) — notification emails do not invite replies. -- `EXTENSIONS_V2_MXROUTE_SERVER`, `EXTENSIONS_V2_MXROUTE_USERNAME`, `EXTENSIONS_V2_MXROUTE_PASSWORD`: Mailbox credentials for the MXroute SMTP API (`https://smtpapi.mxroute.com/`). Required when `EXTENSIONS_V2_EMAIL_PROVIDER=mxroute`. +- `EXTENSIONS_V2_MXROUTE_SERVER`, `EXTENSIONS_V2_MXROUTE_USERNAME`: Dashboard-managed runtime variables identifying the mailbox for the MXroute SMTP API (`https://smtpapi.mxroute.com/`). Required when `EXTENSIONS_V2_EMAIL_PROVIDER=mxroute`. +- `EXTENSIONS_V2_MXROUTE_PASSWORD`: Mailbox password, stored as a secret. Required when `EXTENSIONS_V2_EMAIL_PROVIDER=mxroute`. - `EXTENSIONS_V2_RESEND_API_KEY`: API key for Resend. Required when `EXTENSIONS_V2_EMAIL_PROVIDER=resend`. - `EXTENSIONS_RETENTION_MODE`: `dry-run` (report only) or `compact` (prune old revision bodies). See [extensions v2 README](src/services/extensions/v2/README.md#revisions-and-retention). - `M2M_ISSUER`, `M2M_JWKS_URL`, `M2M_AUDIENCE`: Issuer, JWKS endpoint, and expected audience for machine-to-machine access tokens minted by the central auth service (`FOSSBilling/auth`). Defaults live in `wrangler.jsonc`; preview environments need them pointed at the issuing preview. See [extensions v2 README](src/services/extensions/v2/README.md#machine-publishing). @@ -100,7 +112,7 @@ npm install ### Local Setup -1. Create a `.dev.vars` file for your secrets: +1. Create a `.dev.vars` file for local runtime variables and secrets: ```env GITHUB_TOKEN="your-token" diff --git a/package-lock.json b/package-lock.json index 213616ae..3785a9f4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,6 +11,9 @@ "@octokit/graphql": "9.0.5", "@octokit/request": "10.0.16", "@scalar/hono-api-reference": "0.12.18", + "@zeropress/svg-hush": "0.1.1", + "@zip.js/zip.js": "2.24.0", + "aws4fetch": "1.0.20", "badge-maker": "6.0.0", "better-auth": "^1.7.14", "drizzle-orm": "0.45.4", @@ -3171,6 +3174,26 @@ "url": "https://opencollective.com/vitest" } }, + "node_modules/@zeropress/svg-hush": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/@zeropress/svg-hush/-/svg-hush-0.1.1.tgz", + "integrity": "sha512-RC19T+8sJVNOJW7iMDqg7SwZfnx9WQ54xRPmzNsj38iq8ZNhibOdA8JAUqZ4FizqTvv1OWvaUmrkNTe1LFxz9Q==", + "license": "MIT", + "engines": { + "node": ">=22.22.0" + } + }, + "node_modules/@zip.js/zip.js": { + "version": "2.24.0", + "resolved": "https://registry.npmjs.org/@zip.js/zip.js/-/zip.js-2.24.0.tgz", + "integrity": "sha512-WlaVKzdqBK+RF3bdiXZWbdCnvGB6PdO4IHC1mn/2Yfl3bASgqUNegnU3+PNg61iecwoPfUi5BgLUPqBukXpyWw==", + "license": "BSD-3-Clause", + "engines": { + "bun": ">=0.7.0", + "deno": ">=1.0.0", + "node": ">=18.0.0" + } + }, "node_modules/acorn": { "version": "8.19.0", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.19.0.tgz", @@ -3230,6 +3253,12 @@ "node": ">=12" } }, + "node_modules/aws4fetch": { + "version": "1.0.20", + "resolved": "https://registry.npmjs.org/aws4fetch/-/aws4fetch-1.0.20.tgz", + "integrity": "sha512-/djoAN709iY65ETD6LKCtyyEI04XIBP5xVvfmNxsEP0uJB5tyaGBztSryRr4HqMStr9R06PisQE7m9zDTXKu6g==", + "license": "MIT" + }, "node_modules/badge-maker": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/badge-maker/-/badge-maker-6.0.0.tgz", diff --git a/package.json b/package.json index e3c360b5..266ad83c 100644 --- a/package.json +++ b/package.json @@ -29,6 +29,9 @@ "@octokit/graphql": "9.0.5", "@octokit/request": "10.0.16", "@scalar/hono-api-reference": "0.12.18", + "@zeropress/svg-hush": "0.1.1", + "@zip.js/zip.js": "2.24.0", + "aws4fetch": "1.0.20", "badge-maker": "6.0.0", "better-auth": "^1.7.14", "drizzle-orm": "0.45.4", diff --git a/src/app/index.ts b/src/app/index.ts index e03fddd5..0f775643 100644 --- a/src/app/index.ts +++ b/src/app/index.ts @@ -1,3 +1,7 @@ +import { + consumePackageJobs, + recoverPackageJobs +} from "../lib/adapters/cloudflare/package-jobs"; import { getExtensionsDb } from "../lib/db"; import { maintainExtensionResources, @@ -75,12 +79,20 @@ app.all("/*", (c) => { }); export default { + queue: consumePackageJobs, fetch: app.fetch, request: app.request.bind(app), scheduled: async ( _event: ScheduledController, env: CloudflareBindings ): Promise => { + try { + await recoverPackageJobs(env); + } catch (error) { + logError("extensions/packages", "Package job recovery failed", { + error: error instanceof Error ? error.message : String(error) + }); + } const db = getExtensionsDb(env.DB_EXTENSIONS); await maintainExtensionResources(db, { mode: env.EXTENSIONS_RETENTION_MODE diff --git a/src/lib/adapters/b2/package-store.ts b/src/lib/adapters/b2/package-store.ts new file mode 100644 index 00000000..09bf1465 --- /dev/null +++ b/src/lib/adapters/b2/package-store.ts @@ -0,0 +1,169 @@ +import { AwsClient } from "aws4fetch"; +import { + PACKAGE_LIMITS, + sha256 +} from "../../../services/extensions/v2/package-builder"; +import { PackageError } from "../../../services/extensions/v2/packages/errors"; +import { + boundedBytes, + type PackageStore +} from "../../../services/extensions/v2/packages/store"; + +// Includes generated manifest and ZIP headers as well as the source contents. +const MAX_STORED_BYTES = PACKAGE_LIMITS.expandedBytes + 1024 * 1024; + +export function b2ObjectStore( + getEnv: (name: string) => string | undefined, + kind: "packages" | "assets" +): PackageStore { + const get = (suffix: string) => { + const value = getEnv(`EXTENSION_${kind.toUpperCase()}_S3_${suffix}`); + if (!value) + throw unavailable("Package storage configuration is incomplete"); + return value; + }; + const origin = new URL(get("ENDPOINT")); + const bucket = get("BUCKET"); + const prefix = + kind === "packages" + ? (getEnv("EXTENSION_PACKAGES_S3_PREFIX") ?? "local") + : ""; + if ( + origin.protocol !== "https:" || + origin.username || + origin.password || + origin.search || + origin.hash || + origin.pathname !== "/" || + !/^[a-zA-Z0-9-]+$/.test(bucket) || + (kind === "packages" && !/^[a-zA-Z0-9_-]+$/.test(prefix)) + ) + throw unavailable("Invalid package storage configuration"); + if (kind === "assets") { + let base: URL; + try { + base = new URL(getEnv("EXTENSION_ASSETS_PUBLIC_URL") ?? ""); + } catch { + throw unavailable("Invalid public asset URL configuration"); + } + if ( + base.protocol !== "https:" || + base.username || + base.password || + base.search || + base.hash + ) + throw unavailable("Invalid public asset URL configuration"); + } + const aws = new AwsClient({ + accessKeyId: get("ACCESS_KEY_ID"), + secretAccessKey: get("SECRET_ACCESS_KEY"), + region: get("REGION"), + service: "s3", + retries: 0 + }); + const uuid = "[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}"; + const pattern = + kind === "assets" + ? new RegExp(`^extensions/${uuid}/icons/${uuid}\\.(?:svg|png|avif)$`) + : new RegExp(`^${uuid}$`); + function target(key: string) { + // Keys come from D1 reservations, never from caller-supplied paths. + if (!pattern.test(key)) throw unavailable("Invalid storage object ID"); + return new URL( + `/${bucket}/${kind === "packages" ? `${prefix}/packages/` : ""}${key}`, + origin + ); + } + async function request(url: URL, init: RequestInit) { + try { + return await fetch( + await aws.sign(url, { + ...init, + redirect: "manual", + signal: AbortSignal.timeout(15000) + }) + ); + } catch { + throw unavailable("Package storage request failed"); + } + } + return { + async put(key, bytes, checksum) { + if ( + bytes.byteLength > MAX_STORED_BYTES || + (await sha256(bytes)) !== checksum + ) + throw unavailable("Package object failed checksum or size validation"); + const response = await request(target(key), { + method: "PUT", + body: bytes, + headers: { + "Content-Type": + kind === "packages" + ? "application/zip" + : key.endsWith(".svg") + ? "image/svg+xml" + : key.endsWith(".avif") + ? "image/avif" + : "image/png", + "Cache-Control": + kind === "packages" + ? "private, no-store" + : "public, max-age=31536000, immutable", + "x-amz-content-sha256": checksum, + "x-amz-meta-sha256": checksum + } + }); + await response.body?.cancel(); + if (!response.ok) throw unavailable("Package upload failed"); + }, + async get(key, checksum) { + const response = await request(target(key), { method: "GET" }); + if (response.status === 404) { + await response.body?.cancel(); + return null; + } + if ( + response.status !== 200 || + (checksum && response.headers.get("x-amz-meta-sha256") !== checksum) + ) { + await response.body?.cancel(); + throw unavailable( + "Stored package is unavailable or has a different checksum" + ); + } + // Verify actual bytes before returning an object. The size + // bound also applies to responses without Content-Length (e.g. chunked). + let bytes: Uint8Array; + try { + bytes = await boundedBytes(response.body, MAX_STORED_BYTES); + } catch { + throw unavailable( + "Stored package could not be read within its size limit" + ); + } + if (checksum && (await sha256(bytes)) !== checksum) + throw unavailable("Stored package failed checksum verification"); + return new Blob([bytes]).stream(); + }, + async delete(key) { + const url = target(key); + const head = await request(url, { method: "HEAD" }); + await head.body?.cancel(); + if (head.status === 404) return; + const version = head.headers.get("x-amz-version-id"); + if (!head.ok || !version) + throw unavailable("Unable to identify package version for cleanup"); + // B2 DELETE without versionId only adds a marker and keeps billable bytes. + url.searchParams.set("versionId", version); + const response = await request(url, { method: "DELETE" }); + await response.body?.cancel(); + if (!response.ok && response.status !== 404) + throw unavailable("Package cleanup failed"); + } + }; +} +function unavailable(message: string) { + return new PackageError("PACKAGE_UNAVAILABLE", message, 503); +} diff --git a/src/lib/adapters/cloudflare/index.ts b/src/lib/adapters/cloudflare/index.ts index 34a22cae..89f80c7b 100644 --- a/src/lib/adapters/cloudflare/index.ts +++ b/src/lib/adapters/cloudflare/index.ts @@ -1,3 +1,4 @@ +import { CloudflareQueueAdapter } from "./queue"; import { IPlatformBindings } from "../../interfaces"; import { CloudflareKVAdapter } from "./cache"; import { CloudflareEnvironmentAdapter } from "./environment"; @@ -18,6 +19,13 @@ export function createCloudflareBindings( let bindings = bindingsCache.get(env); if (!bindings) { bindings = { + queues: env.EXTENSION_PACKAGE_QUEUE + ? { + EXTENSION_PACKAGE_QUEUE: new CloudflareQueueAdapter( + env.EXTENSION_PACKAGE_QUEUE + ) + } + : {}, caches: { CACHE_KV: new CloudflareKVAdapter(env.CACHE_KV), AUTH_KV: new CloudflareKVAdapter(env.AUTH_KV) diff --git a/src/lib/adapters/cloudflare/package-assets.ts b/src/lib/adapters/cloudflare/package-assets.ts new file mode 100644 index 00000000..82b2e283 --- /dev/null +++ b/src/lib/adapters/cloudflare/package-assets.ts @@ -0,0 +1,63 @@ +import { PackageBuildDatabase } from "../../../services/extensions/v2/db/package-builds"; +import { logError } from "../../logger"; +import { revalidateExtensions } from "../../../services/extensions/v2/revalidate"; +import type { Context } from "hono"; +import { getPlatform } from "../../middleware"; +import { PackageError } from "../../../services/extensions/v2/packages/errors"; + +// Publishing changes the listing URL only; icons are already stored in B2. +async function updatePublishedIconUrl( + c: Context<{ Bindings: CloudflareBindings }>, + extensionId: string +) { + extensionId = extensionId.toLowerCase(); + const platform = getPlatform(c); + const repository = new PackageBuildDatabase(c.env.DB_EXTENSIONS); + const row = await repository.publishedIcon(extensionId); + if (!row?.object_key) return; + const base = platform.getEnv("EXTENSION_ASSETS_PUBLIC_URL"); + if (!base) + throw new PackageError( + "ASSETS_UNAVAILABLE", + "Public asset URL is not configured", + 503 + ); + const publicBase = new URL(base); + if ( + publicBase.protocol !== "https:" || + publicBase.username || + publicBase.password || + publicBase.search || + publicBase.hash + ) + throw new PackageError( + "ASSETS_UNAVAILABLE", + "Invalid public asset URL", + 503 + ); + const iconUrl = `${publicBase.href.replace(/\/$/, "")}/${row.object_key}`; + return repository.updatePublishedIcon( + extensionId, + row.published_revision_id, + iconUrl + ); +} + +export function schedulePublishedIconUpdate( + c: Context<{ Bindings: CloudflareBindings }>, + extensionId: string +) { + if (!getPlatform(c).getEnv("EXTENSION_ASSETS_S3_ENDPOINT")) return; + c.executionCtx.waitUntil( + updatePublishedIconUrl(c, extensionId) + .then((changed) => { + if (changed) revalidateExtensions(c); + }) + .catch((error) => { + logError("extensions/packages", "Published icon URL update failed", { + extensionId, + error: error instanceof Error ? error.message : String(error) + }); + }) + ); +} diff --git a/src/lib/adapters/cloudflare/package-images.ts b/src/lib/adapters/cloudflare/package-images.ts new file mode 100644 index 00000000..9b08c0ea --- /dev/null +++ b/src/lib/adapters/cloudflare/package-images.ts @@ -0,0 +1,62 @@ +import { sanitizeSvg } from "@zeropress/svg-hush/workerd"; +import type { + PackageImageProcessor, + IconVariant +} from "../../../services/extensions/v2/packages/images"; +import { PackageError } from "../../../services/extensions/v2/packages/errors"; +import { boundedBytes } from "../../../services/extensions/v2/packages/store"; + +export class CloudflarePackageImages implements PackageImageProcessor { + constructor(private readonly images?: ImagesBinding) {} + async prepare( + icon: Uint8Array, + isSvg: boolean + ): Promise { + const images = this.images; + if (isSvg) { + try { + return [{ format: "svg", bytes: new Uint8Array(sanitizeSvg(icon)) }]; + } catch { + throw new PackageError( + "INVALID_ICON", + "Package SVG icon could not be sanitized", + 422 + ); + } + } + if (!images) + throw new PackageError( + "PACKAGES_DISABLED", + "Image processing is not configured", + 503 + ); + const variants: IconVariant[] = []; + try { + const info = await images.info(new Blob([icon]).stream()); + // Check decoded format too: a misleading filename must not bypass sanitization. + if (info.format === "image/svg+xml") + return [{ format: "svg", bytes: new Uint8Array(sanitizeSvg(icon)) }]; + for (const format of ["avif", "png"] as const) { + const output = await images + .input(new Blob([icon]).stream()) + .transform({ + width: 256, + height: 256, + fit: "scale-down" + }) + .output({ format: `image/${format}`, anim: false }); + variants.push({ + format, + bytes: await boundedBytes(output.image(), 1024 * 1024) + }); + } + } catch { + throw new PackageError( + "INVALID_ICON", + "Package icon could not be processed", + 422 + ); + } + return variants; + } +} diff --git a/src/lib/adapters/cloudflare/package-jobs.ts b/src/lib/adapters/cloudflare/package-jobs.ts new file mode 100644 index 00000000..8e50757e --- /dev/null +++ b/src/lib/adapters/cloudflare/package-jobs.ts @@ -0,0 +1,87 @@ +import { logError } from "../../logger"; +import { createCloudflareBindings } from "./index"; +import { PackageJobsDatabase } from "../../../services/extensions/v2/db/package-jobs"; +import { PackageBuildDatabase } from "../../../services/extensions/v2/db/package-builds"; +import { + prepareQueuedPackage, + type PackagePreparation +} from "../../../services/extensions/v2/packages/prepare"; +import { + processPackageJob, + dispatchPackageJob +} from "../../../services/extensions/v2/packages/jobs"; +import { PackageError } from "../../../services/extensions/v2/packages/errors"; +import { configuredPackageStore } from "./package-store"; +import { CloudflarePackageImages } from "./package-images"; + +export function packagePreparation( + env: CloudflareBindings +): PackagePreparation { + const bindings = createCloudflareBindings(env); + const getEnv = (name: string) => bindings.environment.get(name); + const store = configuredPackageStore(env, getEnv); + const baseUrl = getEnv("EXTENSION_PACKAGES_BASE_URL"); + if (!store || !baseUrl) + throw new PackageError( + "PACKAGES_DISABLED", + "Package storage or download URL is not configured", + 503 + ); + return { + repository: new PackageBuildDatabase(env.DB_EXTENSIONS), + store, + images: new CloudflarePackageImages(env.PACKAGE_IMAGES), + baseUrl + }; +} + +export async function consumePackageJobs( + batch: MessageBatch<{ jobId: string }>, + env: CloudflareBindings +) { + const jobs = new PackageJobsDatabase(env.DB_EXTENSIONS); + for (const message of batch.messages) { + if ( + !message.body || + typeof message.body.jobId !== "string" || + !/^[a-f0-9-]{36}$/.test(message.body.jobId) + ) { + message.ack(); + continue; + } + try { + const result = await processPackageJob(jobs, message.body.jobId, (job) => + prepareQueuedPackage(packagePreparation(env), job) + ); + if (result === "busy") message.retry({ delaySeconds: 1200 }); + else if (result === "retry") message.retry({ delaySeconds: 60 }); + else message.ack(); + } catch { + message.retry({ delaySeconds: 60 }); + } + } +} + +export async function recoverPackageJobs(env: CloudflareBindings) { + const mode = createCloudflareBindings(env).environment.get( + "EXTENSION_PACKAGES_MODE" + ); + if ( + !env.EXTENSION_PACKAGE_QUEUE || + !["local", "enabled"].includes(mode ?? "") + ) + return; + const jobs = new PackageJobsDatabase(env.DB_EXTENSIONS); + await jobs.expire(); + const queue = createCloudflareBindings(env).queues!.EXTENSION_PACKAGE_QUEUE; + for (const id of await jobs.pending()) { + try { + await dispatchPackageJob(jobs, queue, id); + } catch (error) { + logError("extensions/packages", "Package job dispatch failed", { + jobId: id, + error: error instanceof Error ? error.message : String(error) + }); + } + } +} diff --git a/src/lib/adapters/cloudflare/package-store.ts b/src/lib/adapters/cloudflare/package-store.ts new file mode 100644 index 00000000..cfc073a8 --- /dev/null +++ b/src/lib/adapters/cloudflare/package-store.ts @@ -0,0 +1,62 @@ +import { PackageError } from "../../../services/extensions/v2/packages/errors"; +import type { PackageStore } from "../../../services/extensions/v2/packages/store"; +import type { Context } from "hono"; +import { b2ObjectStore } from "../b2/package-store"; +import { getPlatform } from "../../middleware"; + +export function packageStore( + c: Context<{ Bindings: CloudflareBindings }> +): PackageStore | null { + return configuredPackageStore(c.env, (name) => getPlatform(c).getEnv(name)); +} + +export function configuredPackageStore( + env: CloudflareBindings, + getEnv: (name: string) => string | undefined +): PackageStore | null { + const mode = getEnv("EXTENSION_PACKAGES_MODE"); + const production = getEnv("ENVIRONMENT") === "production"; + if (mode !== "enabled" && !(mode === "local" && !production)) return null; + if (production) { + const base = getEnv("EXTENSION_PACKAGES_BASE_URL"); + let url: URL; + try { + url = new URL(base ?? ""); + } catch { + throw new PackageError( + "PACKAGES_DISABLED", + "Package download URL is not configured", + 503 + ); + } + if ( + url.protocol !== "https:" || + url.username || + url.password || + url.search || + url.hash || + url.pathname !== "/extensions/v2" || + !getEnv("EXTENSION_PACKAGES_S3_ENDPOINT") || + !getEnv("EXTENSION_PACKAGES_S3_PREFIX") || + getEnv("EXTENSION_PACKAGES_S3_PREFIX") === "local" || + !getEnv("EXTENSION_ASSETS_S3_ENDPOINT") || + !getEnv("EXTENSION_ASSETS_PUBLIC_URL") || + !env.PACKAGE_IMAGES + ) { + throw new PackageError( + "PACKAGES_DISABLED", + "Production package storage, images or download URL is not configured", + 503 + ); + } + } + const packages = b2ObjectStore(getEnv, "packages"); + const assets = b2ObjectStore(getEnv, "assets"); + const storeFor = (key: string) => + key.startsWith("extensions/") ? assets : packages; + return { + put: (key, bytes, checksum) => storeFor(key).put(key, bytes, checksum), + get: (key, checksum) => storeFor(key).get(key, checksum), + delete: (key) => storeFor(key).delete(key) + }; +} diff --git a/src/lib/adapters/cloudflare/queue.ts b/src/lib/adapters/cloudflare/queue.ts new file mode 100644 index 00000000..495d5aea --- /dev/null +++ b/src/lib/adapters/cloudflare/queue.ts @@ -0,0 +1,9 @@ +import type { IMessageQueue } from "../../interfaces"; + +export class CloudflareQueueAdapter implements IMessageQueue { + constructor(private readonly queue: Queue) {} + + async send(body: unknown): Promise { + await this.queue.send(body, { contentType: "json" }); + } +} diff --git a/src/lib/context.ts b/src/lib/context.ts index 1bed40fd..5210bb67 100644 --- a/src/lib/context.ts +++ b/src/lib/context.ts @@ -1,7 +1,8 @@ import { Context } from "hono"; -import { ICache, IPlatformBindings } from "./interfaces"; +import { ICache, IMessageQueue, IPlatformBindings } from "./interfaces"; export interface PlatformContext { + getQueue(name: string): IMessageQueue; getCache(name: string): ICache; getEnv(key: string): string | undefined; readonly raw: Context; @@ -12,6 +13,11 @@ export function createPlatformContext( bindings: IPlatformBindings ): PlatformContext { return { + getQueue(name: string): IMessageQueue { + const queue = bindings.queues?.[name]; + if (!queue) throw new Error(`Queue binding '${name}' not found`); + return queue; + }, getCache(name: string): ICache { const cache = bindings.caches[name]; if (!cache) { diff --git a/src/lib/interfaces.ts b/src/lib/interfaces.ts index dfdfc75d..a4523baa 100644 --- a/src/lib/interfaces.ts +++ b/src/lib/interfaces.ts @@ -21,7 +21,12 @@ export interface IEnvironment { get(key: string): string | undefined; } +export interface IMessageQueue { + send(body: unknown): Promise; +} + export interface IPlatformBindings { + queues?: Record; caches: Record; environment: IEnvironment; } diff --git a/src/lib/releases.ts b/src/lib/releases.ts index b3093057..6548f596 100644 --- a/src/lib/releases.ts +++ b/src/lib/releases.ts @@ -1,7 +1,9 @@ -import { rcompare, valid } from "semver"; +import { prerelease, rcompare, valid } from "semver"; export interface ReleaseTag { tag: string; + date?: string; + is_latest?: boolean; } // Newest first by semver. Tags that aren't valid semver can't be ordered @@ -22,3 +24,29 @@ export function sortReleasesDescending( ...withInvalidTag ]; } + +// Explicit selection wins; older records without a selection prefer stable versions. +export function latestRelease( + releases: readonly T[] +): T | undefined { + const selected = releases.find((release) => release.is_latest === true); + if (selected) return selected; + const stable = releases.filter( + (release) => valid(release.tag) && prerelease(release.tag) === null + ); + return (stable.length ? stable : releases).reduce( + (latest, release) => { + if (!latest) return release; + const a = valid(release.tag), + b = valid(latest.tag); + const order = a && b ? rcompare(a, b) : a ? -1 : b ? 1 : 0; + if (order) return order < 0 ? release : latest; + const time = (date?: string) => { + const value = Date.parse(date ?? ""); + return Number.isFinite(value) ? value : -Infinity; + }; + return time(release.date) > time(latest.date) ? release : latest; + }, + undefined + ); +} diff --git a/src/services/extensions/v1/README.md b/src/services/extensions/v1/README.md index 8be81084..092f3da5 100644 --- a/src/services/extensions/v1/README.md +++ b/src/services/extensions/v1/README.md @@ -11,11 +11,13 @@ FOSSBilling 0.8.9 still uses v1. Upgrade integrations to [extensions v2](../v2/R ## Migration -| v1 endpoint | v2 replacement | -| ------------------- | ----------------------------------------------------------------------------- | -| `GET /list` | `GET /extensions/v2/extensions` | -| `GET /{id}` | `GET /extensions/v2/extensions/{id}` | -| `GET /{id}/version` | Read the latest release's tag (`releases[0].tag`) from the v2 detail response | +| v1 endpoint | v2 replacement | +| ------------------- | ------------------------------------------------------------------------------------------------------------- | +| `GET /list` | `GET /extensions/v2/extensions` | +| `GET /{id}` | `GET /extensions/v2/extensions/{id}` | +| `GET /{id}/version` | Read `version` from the v2 detail response (the selected latest release, or newest stable release by default) | + +The v2 version replacement returns JSON rather than plain text. Its `version` is stored without a leading `v`, while the v1 version endpoint preserves the release tag. For oversized published content, the v2 detail endpoint returns `409 CONTENT_UNAVAILABLE` instead of a version; handle that error before reading `result.version`. V2 lists use cursor pagination, not v1 limit/offset pagination. Use type `module` instead of `mod`. Releases carry full `requires.fossbilling` constraints; consumers must evaluate that constraint rather than a derived minimum. diff --git a/src/services/extensions/v1/database.ts b/src/services/extensions/v1/database.ts index a7f66e79..b120c58c 100644 --- a/src/services/extensions/v1/database.ts +++ b/src/services/extensions/v1/database.ts @@ -3,7 +3,7 @@ import { and, eq, isNotNull, isNull, sql } from "drizzle-orm"; import { ExtensionsDb } from "../../../lib/db"; import { extensions, developers } from "../v2/db/schema"; import { DatabaseResult } from "../../../lib/interfaces"; -import { sortReleasesDescending } from "../../../lib/releases"; +import { latestRelease, sortReleasesDescending } from "../../../lib/releases"; import { parseJSON } from "../../../lib/json"; import type { Release as ManifestRelease } from "../v2/schemas/extensions"; import { Extension, Release, Author, Repository } from "./interfaces"; @@ -190,10 +190,10 @@ export class ExtensionsDatabase { ) ); if (!row) return this.notFound(id); - const releases = sortReleasesDescending( + const release = latestRelease( JSON.parse(row.releases!) as ManifestRelease[] ); - return { data: { version: releases[0]?.tag ?? null }, error: null }; + return { data: { version: release?.tag ?? null }, error: null }; } catch (error) { return this.databaseError(error); } diff --git a/src/services/extensions/v1/interfaces.ts b/src/services/extensions/v1/interfaces.ts index 79429486..1bbca993 100644 --- a/src/services/extensions/v1/interfaces.ts +++ b/src/services/extensions/v1/interfaces.ts @@ -54,6 +54,7 @@ export type User = { export type Release = { tag: string; + is_latest?: boolean; date: string; download_url: string; changelog_url?: string; diff --git a/src/services/extensions/v2/PACKAGING.md b/src/services/extensions/v2/PACKAGING.md new file mode 100644 index 00000000..7cabbd67 --- /dev/null +++ b/src/services/extensions/v2/PACKAGING.md @@ -0,0 +1,284 @@ +# Directory-built packages + +Authors upload ready-to-install distribution ZIPs, with any compilation already +complete. The API validates and repacks them with directory-generated manifests, +then publishes them through the existing revision review. It never runs uploaded +code or package build scripts. ZIPs use the private B2 bucket; prepared icons use +the public asset bucket, including icons for pending submissions. Automated tests +use the B2 adapter with mocked S3 responses. +Package routes are disabled by default. Local development and production enablement are separate modes. + +## Local development and testing + +Run the API through the normal `npm run dev` command. Use ignored `.dev.vars` +for local credentials, set `ENVIRONMENT=local` and `EXTENSION_PACKAGES_MODE=local`, +and set `EXTENSION_PACKAGES_BASE_URL` to the local API's `/extensions/v2` URL. +Apply local migrations with `npm run db:migrate:all:local`. The configured Queue +and Images bindings run locally under Wrangler; B2 requests use the configured +buckets and development keys, so use a separate key and package object prefix. + +`npm run test:all` covers the package lifecycle through the real API routes and +D1 repository, with mocked S3 responses and emulated image processing. Node tests +also exercise package preparation and B2 storage without a Cloudflare runtime. + +## API and review + +The OpenAPI reference is available at `/extensions/v2/docs`. + +- `POST /extensions/v2/packages` accepts manifest v1 JSON and either a ready-to-install + ZIP or an existing `source_id` in a multipart form. It stores the upload privately + and returns **202** with a job ID, status and private status URL in `Location`. + An active owning developer profile is required. Machine publishers need + `extensions:push` and an active mapped developer binding. Moderators may + rebuild published stored source for a correction, but cannot upload new files + for another developer. +- `GET /extensions/v2/package-jobs/{jobId}` returns `queued`, `processing`, `ready` + or `failed`. Ready jobs include the artifact; failed jobs include an actionable + error. Reads require the current owning developer/publisher or a moderator, + and are never publicly cached. Identical requests reuse queued/completed work. +- `GET /extensions/v2/packages/{packageId}` returns the artifact metadata. + `/download` serves its ZIP and `/icon` serves its prepared display image. + Before approval, only the current owner and moderators have access. Delisting + removes public access. +- `POST /extensions/v2/packages/cleanup` lets moderators reclaim unreferenced + object reservations older than one day. + +Releases carry `artifact_id`, `sha256`, the download URL and manifest requirements. +Submission and approval validate these against the stored artifact. The selected +latest release also supplies the overview's version, download, requirements and +manifest-backed metadata. Editing a release requires its stored package or a +source ZIP; the directory always queues package preparation before saving it. + +At most one release can have `is_latest: true`. Without a selection, the newest +stable release is used, falling back to a prerelease. The directory's release +editor provides a “Set as Latest Release” checkbox, checked by default for new +stable releases. This flag stays out of the manifest. + +## Queue processing and provider boundaries + +Queue messages contain only job IDs. D1 stores the validated manifest, captured +owner/profile generation/ownership epoch, upload reference and processing result. +The consumer rechecks account ownership and machine-publisher revocation before +processing. The staged upload becomes the immutable source object after validation, +so it is not uploaded to package storage a second time. + +The generic `IMessageQueue` producer interface lives in `src/lib/interfaces.ts`; +`CloudflareQueueAdapter` implements it. `processPackageJob` depends on a +`PackageJobRepository` and a preparation callback, not Cloudflare MessageBatch, +acknowledgements or retry APIs. Package preparation accepts a `PackageRepository`, `PackageStore` and +`PackageImageProcessor`. SQL and guarded writes live in `db/package-builds.ts`; +Cloudflare bindings, queue delivery and image processing live under +`src/lib/adapters/cloudflare/`, and the B2 store lives under `src/lib/adapters/b2/`. +The package workflow contains no Cloudflare binding types or SDK calls. A different +runtime supplies these interfaces and its own queue consumer; the rest of the API +still has existing platform integrations outside this package workflow. + +Wrangler declares the production producer/consumer pair; local development emulates it. The +consumer handles one message at a time, with concurrency limited to one. Duplicate +delivery uses a 20-minute processing lease; a busy duplicate is delayed until that +lease can expire. Temporary failures retry after a minute, at most three processing +attempts. Invalid archives and revoked ownership fail immediately. The normal +scheduled handler redispatches unsent or stale jobs; this covers interruption +between saving a job and publishing its queue message. It does not rely on the +browser or the HTTP request remaining connected. + +Jobs interrupted or undelivered for a day are marked expired once no active lease +remains. Terminal jobs are retained for seven days, with at most 256 jobs per owning +account. Cleanup never deletes uploads referenced by active jobs, and detaches +terminal job references before reclaiming abandoned objects. Published artifacts +remain immutable and are not removed with job history. + +The directory polls private job status and retains a release-job reference across +reloads. The publisher waits for completion before submitting a revision; failed +or timed-out jobs stop publication, and a retry reuses existing work. Server-side +new submissions and manifest corrections also poll with a bounded wait. + +## Storage and rebuilds + +Each developer profile generation, extension ID and version has one immutable +source ZIP. Changing any source bytes, including repacking, requires a new +version. Manifest changes produce a new immutable artifact from that source; +directory-only changes such as README edits keep the existing package. +Approval publishes the revision and its artifacts in one D1 transaction. Older +approved artifacts retain their URLs and checksums. + +Object reservations precede uploads. Unique constraints deduplicate concurrent +builds; failed writes remain recorded for cleanup. Cleanup claims unreferenced +objects before deletion, and builds cannot reference claimed objects. The +per-account allowance is 256 objects / 512 MiB, including abandoned reservations +and prepared images. Ready sources and artifacts are retained. + +ZIPs must contain ready-to-install files at the root. Packaging replaces the +optional root `manifest.json`. Other files are decompressed for CRC and actual +size validation, then their original compressed payloads are copied unchanged +with normalized ZIP headers. Only the generated manifest is compressed. It never runs +scripts from the archive. Input is limited to 5 MiB, 500 entries and 16 MiB of +actual decompressed data; multipart requests are capped at 6 MiB before parsing. +Paths are limited to 1,024 characters and 32 components. +Encrypted entries, symlinks, unsafe paths, duplicate/colliding names, file/directory +conflicts and corrupt archives are rejected. Checksums cover the final stored ZIP. + +## Icons + +Include at most one root `icon.svg`, `icon.png`, `icon.webp`, or `icon.avif`. +There is no manifest icon field; packages without an icon use the generic icon. +Packaging prepares display images once and stores them in the configured public asset bucket. +The original icon stays unchanged in the ZIP. A new extension receives its permanent +UUIDv7 draft identity after package validation, before the first icon is stored. +The first review submission reuses that draft identity. + +SVGs are sanitized with Cloudflare svg-hush through the pinned `@zeropress/svg-hush` +Workers adapter and served directly. SVG inputs are bounded by the ZIP expansion +limit; there is no separate SVG size cap. Raster inputs use the local `PACKAGE_IMAGES` binding to produce +AVIF and PNG variants within 256 × 256 pixels, preserving proportions and +transparency without enlarging small images. Requests prefer SVG when present, +otherwise AVIF when explicitly accepted, with PNG fallback. They read stored +images directly and never reopen a ZIP. + +All prepared icons, including pending ones, use +`extensions//icons/.` in the public asset bucket. +One asset UUID identifies all format variants; manifest corrections reuse them. +UUID paths are unlisted, not access control: anyone given the direct URL can read +the image before approval. ZIPs and package metadata remain private until approved. +The API preview route still checks ownership and returns private cache headers, +`nosniff`, and a sandboxed Content Security Policy. + +Icons are uploaded during packaging, with immutable public cache headers. An asset +upload failure fails the job; it never switches to another storage backend. +Approval updates the listing to the existing CDN URL and performs no image copy. +The revision ID is checked before changing the listing URL to avoid overwriting a +newer publication. + +## Private B2 package testing + +Set these in the integrated local API's ignored `.dev.vars`. The directory +worktree's root `.dev.vars` may hold the development credentials, but the API +process must receive them in its own configuration file. + +```dotenv +EXTENSION_PACKAGES_S3_ENDPOINT="https://s3..backblazeb2.com" +EXTENSION_PACKAGES_S3_REGION="" +EXTENSION_PACKAGES_S3_BUCKET="your-development-packages-bucket" +EXTENSION_PACKAGES_S3_ACCESS_KEY_ID="your-keyID" +EXTENSION_PACKAGES_S3_SECRET_ACCESS_KEY="your-applicationKey" +EXTENSION_PACKAGES_S3_PREFIX="local" +``` + +Keep this bucket private. Use a separate development key restricted to the bucket +with read, write and delete-file permissions. No public URL or domain is required. +Use `EXTENSION_PACKAGES_MODE=local` only outside production. The committed production configuration remains disabled. +Tests run the B2 adapter against mocked S3 responses. Local development uses B2 +with development credentials. An incomplete S3 configuration or upstream +failure returns 503 rather than silently switching storage backends. + +The B2 adapter signs uploads with their SHA-256 payload, stores checksum metadata +and verifies both metadata and downloaded bytes on reads. + +Source and built ZIPs use `local/packages/`. Object IDs are allocated +once by D1 before upload. Uploads carry a signed SHA-256 payload and private cache +headers. Reads are bounded to 17 MiB and verify artifact bytes before returning a +ZIP through the existing permission-checked download route. B2 URLs and credentials +are never returned to visitors. Images use the separate public asset bucket and +its CDN hostname, including before approval. + +Cleanup identifies the stored B2 version and deletes it explicitly. Deleting only +by name would leave billable versions behind. Failed deletion retains the D1 +reservation for retry. + +## B2 asset and Cloudflare testing + +Add these to the local API's ignored `.dev.vars`: + +```dotenv +EXTENSION_ASSETS_S3_ENDPOINT="https://s3..backblazeb2.com" +EXTENSION_ASSETS_S3_REGION="" +EXTENSION_ASSETS_S3_BUCKET="your-development-assets-bucket" +EXTENSION_ASSETS_S3_ACCESS_KEY_ID="your-keyID" +EXTENSION_ASSETS_S3_SECRET_ACCESS_KEY="your-applicationKey" +EXTENSION_ASSETS_PUBLIC_URL="https://extensions-assets.fossbilling.net" +``` + +Restrict the asset key to this bucket with read, write and delete-file permissions. +Pending API previews read these stored objects; cleanup deletes abandoned versions. +Tests mock S3 responses without accessing a real bucket. Public object keys use +`extensions//icons/.`, for example: + +```text +https://extensions-assets.fossbilling.net/extensions/019a0000-0000-7000-8000-000000000001/icons/11111111-1111-4111-8111-111111111111.svg +``` + +The parent is the extension record's UUID, not its public identifier. The icon +record allocates an immutable asset UUID when its display images are prepared. +All format variants share that identity, and manifest-only edits reuse the same image URL. +A different source gets a new asset UUID. Uploads set +`Cache-Control: public, max-age=31536000, immutable` and retain each file's SHA-256 +in object metadata for verification. +Package source records retain the prepared image keys, so retries and metadata +corrections reuse existing images. Directory pages use direct CDN images with AVIF/PNG `` fallback, +bypassing the directory image transformer. + +The bucket is public. The hostname-scoped Cloudflare response-header rule +`extension_assets_security_headers` adds these headers, including on cached +responses: + +```text +Content-Security-Policy: sandbox; default-src 'none'; style-src 'unsafe-inline'; img-src data: +X-Content-Type-Options: nosniff +``` + +Prepared assets enter this bucket during packaging, before approval. Delisting +stops access through the API, but does not revoke direct image URLs or cached copies. +Keep B2 daily caps and alerts configured for download and transaction charges. + +## Production configuration (prepared locally) + +The top-level Wrangler configuration declares bindings, public URLs and deployment +controls, keeping `EXTENSION_PACKAGES_MODE=disabled`. B2 endpoints, regions, bucket +names and the package prefix are dashboard-managed runtime variables. Configure +these under the Worker's **Settings > Variables and Secrets**, not build settings: + +| Runtime variables | Purpose | +| ------------------------------------------------------------------------------------------------ | -------------------------------------------------------------------- | +| `EXTENSION_PACKAGES_S3_ENDPOINT`, `EXTENSION_PACKAGES_S3_REGION`, `EXTENSION_PACKAGES_S3_BUCKET` | Private ZIP storage connection | +| `EXTENSION_PACKAGES_S3_PREFIX` | ZIP object prefix; production requires an explicit non-`local` value | +| `EXTENSION_ASSETS_S3_ENDPOINT`, `EXTENSION_ASSETS_S3_REGION`, `EXTENSION_ASSETS_S3_BUCKET` | Public image storage connection | + +`keep_vars: true` preserves these values during Wrangler deployments. Do not +reintroduce them in the committed `vars` map. Keep development values in ignored +`.dev.vars` files; the examples above use placeholders. No production resources +or secrets are created by a local build or type-generation command. + +Before deployment (including an automatic deployment after merge): + +1. Provision the `extension-packages` Queue. + ZIPs use private B2 and prepared icons use the public B2 asset bucket. + The queue binding must exist even while package mode is disabled; that flag blocks + application routes, not deployment-time binding validation. +2. Supply `EXTENSION_PACKAGES_S3_ACCESS_KEY_ID` and + `EXTENSION_PACKAGES_S3_SECRET_ACCESS_KEY` as Worker secrets for + the private package bucket. Supply separate `EXTENSION_ASSETS_S3_ACCESS_KEY_ID` + and `EXTENSION_ASSETS_S3_SECRET_ACCESS_KEY` for the public asset bucket. + Development keys in `.dev.vars` are not production secrets. +3. Confirm the Images binding is enabled for the account, the scoped CDN security + headers remain configured, and B2 caps/alerts are in place. +4. Apply the package and job tables through the normal D1 deployment workflow. +5. After queue-consumer capacity and moderation requirements are agreed, explicitly set + `EXTENSION_PACKAGES_MODE=enabled` in the reviewed production configuration. + Verify a production smoke test before enabling the dependent directory and + modules publisher changes. Those callers require an enabled, configured API; + merging a disabled API alone does not make package submission available. + +Production requires HTTPS download URLs with the `/extensions/v2` base path, +complete credentials for both B2 buckets, an explicit non-`local` ZIP object prefix, +a public asset URL and the Images binding. An incomplete configuration fails with 503; +it cannot fall back to local ZIP storage. Private package objects use the configured prefix. +Public image objects use the UUID-based layout described above. + +The disabled mode blocks package reads as well as writes. Once managed releases +are published, disabling it also stops their API downloads and private previews; +it is not an upload-only pause switch. Prepared CDN images remain accessible independently, even for pending packages. + +Before enabling package mode, measure admission and queue-consumer invocations +separately: CPU and wall duration, memory failures, queue delay, retries, and +storage/image-service latency. Include near-limit ZIPs and SVG/raster icons. +Local functional tests do not establish production capacity or billing. diff --git a/src/services/extensions/v2/README.md b/src/services/extensions/v2/README.md index 7c476811..30162df1 100644 --- a/src/services/extensions/v2/README.md +++ b/src/services/extensions/v2/README.md @@ -17,13 +17,17 @@ Endpoints are not listed here. The service publishes its own contract: - **OpenAPI document:** `GET /extensions/v2/docs/openapi.json` - **Reference UI:** `GET /extensions/v2/docs` +## Directory Identity + +Extensions and developers have generated UUIDv7 `id` primary keys and separate, globally unique `identifier` fields. Database relationships and machine publisher bindings use UUIDs. Public profile/extension paths use identifiers; `name` is the editable display name. Creation requests supply `identifier`, never the internal ID. The package manifest's `id` remains the public extension identifier. Staged package sources record this explicitly as `extension_identifier`, because a package can be prepared before its directory listing exists. + ## Manifest contract New v2 writes use the [FOSSBilling manifest v1 fields](https://github.com/FOSSBilling/modules/blob/main/manifest.schema.json): `manifest_version: 1`, `type: "module"` (replacing `mod`; standalone `hook` entries require manual reclassification), `author: {name, url?}`, a string `license`, optional `homepage_url` (replacing `website`), and `requires: {fossbilling, php, modules?}`. IDs allow lowercase letters, digits, underscores, and hyphens, up to 64 characters. Names are limited to 100 characters, descriptions to 500, and versions use SemVer without a `v` prefix. -`icon_url` is an optional path relative to the package root, as in `icon.svg`; it cannot be an absolute URL, absolute path, or contain `..` or backslashes. Consumers resolve package assets against their trusted distribution source. Historical icons may still be absolute URLs. +Icons are discovered by convention: include one of `icon.svg`, `icon.png`, `icon.webp`, or `icon.avif` at the package root. The manifest and publishing input have no icon field. Public responses may include a computed `icon_url` for display; it is not editable. Directory packages without an icon return a generic fallback. -API publishing fields `readme`, `source`, `download_url`, and `releases` remain alongside the manifest metadata. Each release carries its own `requires` object instead of `min_fossbilling_version`. Constraints are retained verbatim, including Composer ranges and module dependencies; package CI remains responsible for semantic constraint/license validation and confirming that icon files exist. The API adds size bounds to metadata (author names: 200 characters; license and requirement strings: 1,000; URLs and icon paths: 2,048). +API publishing fields `readme`, `source`, `download_url`, and `releases` remain alongside the manifest metadata. Each release carries its own `requires` object instead of `min_fossbilling_version`. Constraints are retained verbatim, including Composer ranges and module dependencies; package CI remains responsible for semantic constraint/license validation. The API adds size bounds to metadata (author names: 200 characters; license and requirement strings: 1,000; URLs: 2,048). `requires.fossbilling` is mandatory. `requires.php` is an optional additional restriction: when omitted, FOSSBilling’s PHP requirements apply; when supplied, both the core and extension constraints must be satisfied. @@ -41,7 +45,7 @@ For example, after checking an affected package's actual requirements, an operat UPDATE extensions SET releases = json_set(releases, '$[0].requires', json('{"fossbilling":">=0.8.9","php":">=8.3"}')) -WHERE id = 'affected-extension'; +WHERE identifier = 'affected-extension'; ``` Use the ID, index, and verified constraints from the preflight investigation; the values above are examples. Re-run the query until it returns no rows. @@ -52,7 +56,7 @@ After applying the prepared migration, run the read-only `db/resource-inventory. Apply the migration and API deployment in a controlled cutover: the old Worker expects object licenses, while the new Worker requires the added metadata column. -This changes the v2 request/response contract: regenerate the Extensions site's client and update CI publishers together with the API rollout. The modules publisher must stop translating `module` to `mod` and send the manifest fields directly. Package-relative icons use v1's empty-string fallback. The v1 adapter continues to return `mod`, `website`, object licenses, and minimum release versions; constraints are converted to their earliest matching SemVer version. A malformed, unsatisfiable, or Composer-only constraint with no representable v1 minimum produces an explicit error identifying the release for manual correction; it is never replaced with `0.0.0`. +This changes the v2 request/response contract: regenerate the Extensions site's client and update CI publishers together with the API rollout. The modules publisher must stop translating `module` to `mod` and send the manifest fields directly. Icons follow the package filename convention. The v1 adapter continues to return `mod`, `website`, object licenses, and minimum release versions; constraints are converted to their earliest matching SemVer version. A malformed, unsatisfiable, or Composer-only constraint with no representable v1 minimum produces an explicit error identifying the release for manual correction; it is never replaced with `0.0.0`. The frontend revalidation tags are `extensions` and `developers`, shared with the Extensions site. Deploy the API and site tag change together. New code calls `revalidateExtensions(c)`. @@ -130,7 +134,7 @@ Assertions use HS256 with issuer `fossbilling-extensions`, audience `fossbilling Machine clients (CI publishers) authenticate with short-lived access tokens from the central auth service's `client_credentials` grant, bound to this API via RFC 8707 `resource=https://api.fossbilling.net`. Tokens are RFC 9068 JWTs (`typ: at+jwt`, EdDSA) carrying `client_id` and `scope`; verification is local via the issuer's JWKS (`M2M_*` variables, root README), so revocation takes effect at token expiry (10 minutes). -Two scopes gate the machine surface: `extensions:push` covers `POST /extensions` and `PUT /extensions/{id}`; `extensions:publish` additionally covers approving a pending revision on the mapped developer's own extension — the one carve-out from moderator-only approval. Machine tokens are answered 403 on account, moderator, and ownership routes; on public reads they are served the anonymous projection. Bindings are bound to the developer's ownership epoch at registration, so an ownership transfer goes stale and a moderator must re-register before the client writes again. +Two scopes gate the machine surface: `extensions:push` covers `POST /extensions`, `PUT /extensions/{id}`, and package builds through `POST /packages`; `extensions:publish` additionally covers approving a pending revision on the mapped developer's own extension — the one carve-out from moderator-only approval. Machine tokens are answered 403 on account, moderator, and ownership routes; on public reads they are served the anonymous projection. Bindings are bound to the developer's ownership epoch at registration, so an ownership transfer goes stale and a moderator must re-register before the client writes again. A moderator binds a client to a developer profile with `PUT /m2m/clients/{client_id}` (`developer_id`, `can_publish`), lists bindings with `GET /m2m/clients`, and revokes with `DELETE /m2m/clients/{client_id}`. A revoked client's tokens still verify but fail authorization on the next write. Each binding also creates a synthetic internal user (`m2m:`) that anchors revision foreign keys; it is not an account and cannot sign in. @@ -207,3 +211,9 @@ Reconcile and re-run; the migration touches nothing before these checks. ## Code layout See `AGENTS.md` for `routes/`, `db/`, `schemas/`, `github/`, and `middleware.ts`. This service is the reference layout for larger services. + +## Directory-built packages (local development) + +See [PACKAGING.md](./PACKAGING.md) for the authenticated upload/build API, +immutable artifacts, moderation integration, local test-data setup and storage +limits. Production package routes remain disabled until storage is configured. diff --git a/src/services/extensions/v2/db/badges.ts b/src/services/extensions/v2/db/badges.ts index 222ee366..c691a6d6 100644 --- a/src/services/extensions/v2/db/badges.ts +++ b/src/services/extensions/v2/db/badges.ts @@ -2,7 +2,7 @@ import { ZodError } from "zod"; import { and, isNotNull, isNull, sql } from "drizzle-orm"; import { ExtensionsDb } from "../../../../lib/db"; import { DatabaseResult } from "../../../../lib/interfaces"; -import { sortReleasesDescending } from "../../../../lib/releases"; +import { latestRelease } from "../../../../lib/releases"; import { BadgeDataSchema } from "../schemas/badges"; import { MAX_CONTENT_BYTES } from "../resource-limits"; import { databaseError } from "./errors"; @@ -51,7 +51,7 @@ export class ExtensionBadgesDatabase { }); return { data: { - latestRelease: sortReleasesDescending(data.releases)[0] ?? null, + latestRelease: latestRelease(data.releases) ?? null, license: data.license }, error: null diff --git a/src/services/extensions/v2/db/extensions.ts b/src/services/extensions/v2/db/extensions.ts index 1dfe7892..a164300d 100644 --- a/src/services/extensions/v2/db/extensions.ts +++ b/src/services/extensions/v2/db/extensions.ts @@ -1,4 +1,5 @@ import { v7 as uuidv7 } from "uuid"; +import { packageIconUrl, validatePackageContent } from "./packages"; import { MAX_CONTENT_BYTES } from "../resource-limits"; import { and, asc, eq, isNotNull, isNull, or, sql } from "drizzle-orm"; import { alias } from "drizzle-orm/sqlite-core"; @@ -625,11 +626,10 @@ export class ExtensionsDatabase { } } - // Creates the extension record and its first pending revision as one - // transaction. The revision insert is gated on `changes() = 1` from the - // preceding statement (SQLite's per-connection changes()), so an id - // collision or a failed ownership guard leaves neither row behind. See - // toD1Statement for why this is raw sql rather than two builder calls. + // Creates the first pending revision, reusing a package draft when present. + // The batch gates the revision insert on changes() = 1, so a conflicting + // identifier or failed ownership guard cannot create a revision. A package + // draft is reused only when it has never been submitted. async create( input: CreateExtensionInput ): Promise< @@ -640,6 +640,13 @@ export class ExtensionsDatabase { let results; try { + const packageError = await validatePackageContent( + this.db.$client, + input.identifier, + input.developerId, + input.content + ); + if (packageError) return { data: null, error: packageError }; const extensionStmt = toD1Statement(this.db.$client, { sql: `INSERT INTO extensions (id, identifier, developer_id, created_by, created_at, updated_at) SELECT ?, ?, d.id, d.owner_user_id, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP @@ -652,7 +659,11 @@ export class ExtensionsDatabase { SELECT COUNT(*) FROM extension_revisions WHERE submitted_by = ? AND status = 'pending' ) < ? - ON CONFLICT DO NOTHING`, + ON CONFLICT (lower(identifier)) DO UPDATE SET updated_at=CURRENT_TIMESTAMP + WHERE extensions.developer_id=excluded.developer_id + AND extensions.published_at IS NULL + AND NOT EXISTS (SELECT 1 FROM extension_revisions r WHERE r.extension_id=extensions.id) + RETURNING id`, params: [ extensionId, input.identifier, @@ -668,19 +679,22 @@ export class ExtensionsDatabase { const revisionStmt = toD1Statement(this.db.$client, { sql: `INSERT INTO extension_revisions (id, extension_id, developer_id, submitted_by, status, content, ownership_epoch) - SELECT ?, ?, ?, ?, 'pending', ?, ? - WHERE changes() = 1`, + SELECT ?, e.id, ?, ?, 'pending', ?, ? FROM extensions e + WHERE lower(e.identifier)=lower(?) AND changes() = 1`, params: [ revisionId, - extensionId, input.developerId, input.submittedBy, JSON.stringify(input.content), - input.ownershipEpoch + input.ownershipEpoch, + input.identifier ] }); - results = await this.db.$client.batch([extensionStmt, revisionStmt]); + results = await this.db.$client.batch<{ id: string }>([ + extensionStmt, + revisionStmt + ]); } catch (error) { return contentAdmissionError("create", error); } @@ -694,7 +708,11 @@ export class ExtensionsDatabase { } return { - data: { id: extensionId, identifier: input.identifier, revisionId }, + data: { + id: results[0].results[0].id, + identifier: input.identifier, + revisionId + }, error: null }; } @@ -739,6 +757,13 @@ export class ExtensionsDatabase { let results; try { + const packageError = await validatePackageContent( + this.db.$client, + input.identifier, + input.developerId, + input.content + ); + if (packageError) return { data: null, error: packageError }; const extensionStmt = toD1Statement(this.db.$client, { sql: `INSERT INTO extensions (id, identifier, developer_id, created_by, created_at, updated_at) SELECT ?, ?, d.id, d.owner_user_id, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP @@ -756,7 +781,11 @@ export class ExtensionsDatabase { SELECT COUNT(*) FROM extension_revisions WHERE submitted_by = ? AND status = 'pending' ) < ? - ON CONFLICT DO NOTHING`, + ON CONFLICT (lower(identifier)) DO UPDATE SET updated_at=CURRENT_TIMESTAMP + WHERE extensions.developer_id=excluded.developer_id + AND extensions.published_at IS NULL + AND NOT EXISTS (SELECT 1 FROM extension_revisions r WHERE r.extension_id=extensions.id) + RETURNING id`, params: [ extensionId, input.identifier, @@ -770,20 +799,23 @@ export class ExtensionsDatabase { const revisionStmt = toD1Statement(this.db.$client, { sql: `INSERT INTO extension_revisions (id, extension_id, developer_id, submitted_by, status, content, ownership_epoch) - SELECT ?, ?, d.id, ?, 'pending', ?, d.ownership_epoch - FROM developers d - WHERE d.id = ? + SELECT ?, e.id, d.id, ?, 'pending', ?, d.ownership_epoch + FROM developers d JOIN extensions e ON e.developer_id=d.id + WHERE d.id = ? AND lower(e.identifier)=lower(?) AND changes() = 1`, params: [ revisionId, - extensionId, input.submittedBy, JSON.stringify(input.content), - input.developerId + input.developerId, + input.identifier ] }); - results = await this.db.$client.batch([extensionStmt, revisionStmt]); + results = await this.db.$client.batch<{ id: string }>([ + extensionStmt, + revisionStmt + ]); } catch (error) { return contentAdmissionError("createAsPublisher", error); } @@ -800,7 +832,11 @@ export class ExtensionsDatabase { } return { - data: { id: extensionId, identifier: input.identifier, revisionId }, + data: { + id: results[0].results[0].id, + identifier: input.identifier, + revisionId + }, error: null }; } @@ -1174,6 +1210,13 @@ export class ExtensionsDatabase { let results; try { + const packageError = await validatePackageContent( + this.db.$client, + id, + undefined, + valid + ); + if (packageError) return { data: null, error: packageError }; const correctStmt = toD1Statement(this.db.$client, { sql: `INSERT INTO extension_revisions (id, extension_id, developer_id, submitted_by, status, content, @@ -1224,7 +1267,7 @@ export class ExtensionsDatabase { JSON.stringify(valid.releases), valid.homepage_url ?? "", JSON.stringify(valid.license), - valid.icon_url ?? null, + packageIconUrl(valid), valid.readme, JSON.stringify(valid.source), valid.version, diff --git a/src/services/extensions/v2/db/migrations/0031_directory_packages.sql b/src/services/extensions/v2/db/migrations/0031_directory_packages.sql new file mode 100644 index 00000000..5c8d8bc8 --- /dev/null +++ b/src/services/extensions/v2/db/migrations/0031_directory_packages.sql @@ -0,0 +1,92 @@ +CREATE TABLE `package_artifacts` ( + `id` text PRIMARY KEY NOT NULL, + `source_id` text NOT NULL, + `manifest` text NOT NULL, + `manifest_hash` text NOT NULL, + `sha256` text NOT NULL, + `object_id` text NOT NULL, + `download_url` text NOT NULL, + `published_at` text, + `created_at` text DEFAULT CURRENT_TIMESTAMP NOT NULL, + FOREIGN KEY (`source_id`) REFERENCES `package_sources`(`id`) ON UPDATE no action ON DELETE no action, + FOREIGN KEY (`object_id`) REFERENCES `package_objects`(`id`) ON UPDATE no action ON DELETE no action +); +--> statement-breakpoint +CREATE UNIQUE INDEX `idx_package_artifact_manifest` ON `package_artifacts` (`source_id`,`manifest_hash`);--> statement-breakpoint +CREATE TABLE `package_objects` ( + `deleting` integer DEFAULT 0 NOT NULL, + `id` text PRIMARY KEY NOT NULL, + `created_by` text NOT NULL, + `bytes` integer NOT NULL, + `created_at` text DEFAULT CURRENT_TIMESTAMP NOT NULL +); +--> statement-breakpoint +CREATE INDEX `idx_package_objects_creator` ON `package_objects` (`created_by`);--> statement-breakpoint +CREATE TABLE `package_sources` ( + `id` text PRIMARY KEY NOT NULL, + `profile_generation` text NOT NULL, + `extension_identifier` text NOT NULL, + `version` text NOT NULL, + `sha256` text NOT NULL, + `object_id` text NOT NULL, + `created_at` text DEFAULT CURRENT_TIMESTAMP NOT NULL, + FOREIGN KEY (`object_id`) REFERENCES `package_objects`(`id`) ON UPDATE no action ON DELETE no action +); +--> statement-breakpoint +CREATE UNIQUE INDEX `idx_package_source_version` ON `package_sources` (`profile_generation`,`extension_identifier`,`version`); +--> statement-breakpoint +CREATE TRIGGER package_sources_immutable BEFORE UPDATE ON package_sources +BEGIN SELECT RAISE(ABORT, 'package_source_immutable'); END; +--> statement-breakpoint +CREATE TRIGGER package_artifacts_immutable BEFORE UPDATE OF id,source_id,manifest,manifest_hash,sha256,object_id,download_url,created_at ON package_artifacts +BEGIN SELECT RAISE(ABORT, 'package_artifact_immutable'); END; +--> statement-breakpoint +CREATE TRIGGER package_artifacts_publication_immutable BEFORE UPDATE OF published_at ON package_artifacts WHEN OLD.published_at IS NOT NULL +BEGIN SELECT RAISE(ABORT, 'package_publication_immutable'); END; +--> statement-breakpoint +CREATE TRIGGER publish_package_artifacts AFTER UPDATE OF published_revision_id ON extensions +WHEN NEW.published_revision_id IS NOT NULL AND NEW.published_revision_id IS NOT OLD.published_revision_id +BEGIN + UPDATE package_artifacts SET published_at=CURRENT_TIMESTAMP + WHERE published_at IS NULL AND id IN ( + SELECT json_extract(value,'$.artifact_id') FROM json_each(NEW.releases) + ) AND source_id IN ( + SELECT s.id FROM package_sources s JOIN developers d ON d.profile_generation=s.profile_generation + WHERE s.extension_identifier=lower(NEW.identifier) AND d.id=NEW.developer_id + ); +END; + +--> statement-breakpoint +CREATE INDEX idx_package_sources_extension ON package_sources(extension_identifier); +--> statement-breakpoint +CREATE UNIQUE INDEX idx_package_sources_object ON package_sources(object_id); +--> statement-breakpoint +CREATE UNIQUE INDEX idx_package_artifacts_object ON package_artifacts(object_id); + +--> statement-breakpoint +CREATE TABLE `package_icons` ( + `source_id` text PRIMARY KEY NOT NULL, + `avif_object_id` text, + `avif_sha256` text, + `png_object_id` text, + `png_sha256` text, + `svg_object_id` text, + `svg_sha256` text, + FOREIGN KEY (`source_id`) REFERENCES `package_sources`(`id`) ON UPDATE no action ON DELETE no action, + FOREIGN KEY (`avif_object_id`) REFERENCES `package_objects`(`id`) ON UPDATE no action ON DELETE no action, + FOREIGN KEY (`png_object_id`) REFERENCES `package_objects`(`id`) ON UPDATE no action ON DELETE no action, + FOREIGN KEY (`svg_object_id`) REFERENCES `package_objects`(`id`) ON UPDATE no action ON DELETE no action +); +--> statement-breakpoint +CREATE TRIGGER package_icons_immutable BEFORE UPDATE ON package_icons +BEGIN SELECT RAISE(ABORT, 'Package icons are immutable'); END; + +--> statement-breakpoint +CREATE TRIGGER package_sources_no_delete BEFORE DELETE ON package_sources +BEGIN SELECT RAISE(ABORT, 'package_source_immutable'); END; +--> statement-breakpoint +CREATE TRIGGER package_artifacts_no_delete BEFORE DELETE ON package_artifacts +BEGIN SELECT RAISE(ABORT, 'package_artifact_immutable'); END; +--> statement-breakpoint +CREATE TRIGGER package_icons_no_delete BEFORE DELETE ON package_icons +BEGIN SELECT RAISE(ABORT, 'package_icons_immutable'); END; diff --git a/src/services/extensions/v2/db/migrations/0032_package_jobs.sql b/src/services/extensions/v2/db/migrations/0032_package_jobs.sql new file mode 100644 index 00000000..8c44a8b9 --- /dev/null +++ b/src/services/extensions/v2/db/migrations/0032_package_jobs.sql @@ -0,0 +1,30 @@ +CREATE TABLE `package_jobs` ( + `id` text PRIMARY KEY NOT NULL, + `fingerprint` text NOT NULL, + `owner_user_id` text NOT NULL, + `owner` text NOT NULL, + `manifest` text NOT NULL, + `source_id` text, + `upload_object_id` text, + `upload_sha256` text, + `status` text DEFAULT 'queued' NOT NULL, + `attempts` integer DEFAULT 0 NOT NULL, + `lease_id` text, + `lease_until` text, + `artifact_id` text, + `error_code` text, + `error_message` text, + `error_status` integer, + `enqueued_at` text, + `created_at` text DEFAULT CURRENT_TIMESTAMP NOT NULL, + `updated_at` text DEFAULT CURRENT_TIMESTAMP NOT NULL, + FOREIGN KEY (`upload_object_id`) REFERENCES `package_objects`(`id`) ON UPDATE no action ON DELETE no action, + FOREIGN KEY (`artifact_id`) REFERENCES `package_artifacts`(`id`) ON UPDATE no action ON DELETE no action, + CONSTRAINT "package_jobs_status_check" CHECK("package_jobs"."status" IN ('queued','processing','ready','failed')) +); +--> statement-breakpoint +CREATE UNIQUE INDEX `idx_package_jobs_fingerprint` ON `package_jobs` (`fingerprint`) WHERE "package_jobs"."status" != 'failed';--> statement-breakpoint +CREATE INDEX `idx_package_jobs_owner` ON `package_jobs` (`owner_user_id`);--> statement-breakpoint +CREATE INDEX `idx_package_jobs_dispatch` ON `package_jobs` (`status`,`enqueued_at`); +--> statement-breakpoint +CREATE INDEX `idx_package_jobs_upload_object` ON `package_jobs` (`upload_object_id`); diff --git a/src/services/extensions/v2/db/migrations/meta/0031_snapshot.json b/src/services/extensions/v2/db/migrations/meta/0031_snapshot.json new file mode 100644 index 00000000..662bc3d1 --- /dev/null +++ b/src/services/extensions/v2/db/migrations/meta/0031_snapshot.json @@ -0,0 +1,1734 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "78430a63-8fd0-4d21-a557-33958d0386f8", + "prevId": "93fb9a02-8924-42e3-a428-55fd316dbfc9", + "tables": { + "claim_verification_budgets": { + "name": "claim_verification_budgets", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_claim_verification_budgets_expiry": { + "name": "idx_claim_verification_budgets_expiry", + "columns": ["expires_at"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "developer_claims": { + "name": "developer_claims", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "developer_id": { + "name": "developer_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "claimant_id": { + "name": "claimant_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "note": { + "name": "note", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "review_note": { + "name": "review_note", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reviewer_id": { + "name": "reviewer_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + }, + "reviewed_at": { + "name": "reviewed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "github_org_verified": { + "name": "github_org_verified", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "github_verification_note": { + "name": "github_verification_note", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "idx_developer_claims_developer": { + "name": "idx_developer_claims_developer", + "columns": ["developer_id"], + "isUnique": false + }, + "idx_developer_claims_claimant": { + "name": "idx_developer_claims_claimant", + "columns": ["claimant_id"], + "isUnique": false + }, + "idx_developer_claims_pending_unique": { + "name": "idx_developer_claims_pending_unique", + "columns": ["developer_id", "claimant_id"], + "isUnique": true, + "where": "\"developer_claims\".\"status\" = 'pending'" + }, + "idx_developer_claims_pending_queue": { + "name": "idx_developer_claims_pending_queue", + "columns": ["created_at"], + "isUnique": false, + "where": "\"developer_claims\".\"status\" = 'pending'" + } + }, + "foreignKeys": { + "developer_claims_developer_id_developers_id_fk": { + "name": "developer_claims_developer_id_developers_id_fk", + "tableFrom": "developer_claims", + "tableTo": "developers", + "columnsFrom": ["developer_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "developer_claims_claimant_id_users_id_fk": { + "name": "developer_claims_claimant_id_users_id_fk", + "tableFrom": "developer_claims", + "tableTo": "users", + "columnsFrom": ["claimant_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "developer_claims_reviewer_id_users_id_fk": { + "name": "developer_claims_reviewer_id_users_id_fk", + "tableFrom": "developer_claims", + "tableTo": "users", + "columnsFrom": ["reviewer_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": { + "developer_claims_status_check": { + "name": "developer_claims_status_check", + "value": "\"developer_claims\".\"status\" IN ('pending', 'approved', 'rejected')" + }, + "developer_claims_github_org_verified_check": { + "name": "developer_claims_github_org_verified_check", + "value": "\"developer_claims\".\"github_org_verified\" IN (0, 1)" + } + } + }, + "developer_history": { + "name": "developer_history", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "developer_id": { + "name": "developer_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "changed_by": { + "name": "changed_by", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "changed_at": { + "name": "changed_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + } + }, + "indexes": { + "idx_developer_history_developer_changed_at": { + "name": "idx_developer_history_developer_changed_at", + "columns": ["developer_id", "changed_at"], + "isUnique": false + }, + "idx_developer_history_account_changed_at": { + "name": "idx_developer_history_account_changed_at", + "columns": ["changed_by", "changed_at"], + "isUnique": false + } + }, + "foreignKeys": { + "developer_history_changed_by_users_id_fk": { + "name": "developer_history_changed_by_users_id_fk", + "tableFrom": "developer_history", + "tableTo": "users", + "columnsFrom": ["changed_by"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "developer_transfers": { + "name": "developer_transfers", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "developer_id": { + "name": "developer_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token_hash": { + "name": "token_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + }, + "expires_at": { + "name": "expires_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "accepted_by": { + "name": "accepted_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "accepted_at": { + "name": "accepted_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "revoked_at": { + "name": "revoked_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "idx_developer_transfers_token": { + "name": "idx_developer_transfers_token", + "columns": ["token_hash"], + "isUnique": true + }, + "idx_developer_transfers_pending": { + "name": "idx_developer_transfers_pending", + "columns": ["developer_id"], + "isUnique": true, + "where": "\"developer_transfers\".\"accepted_at\" IS NULL AND \"developer_transfers\".\"revoked_at\" IS NULL" + } + }, + "foreignKeys": { + "developer_transfers_developer_id_developers_id_fk": { + "name": "developer_transfers_developer_id_developers_id_fk", + "tableFrom": "developer_transfers", + "tableTo": "developers", + "columnsFrom": ["developer_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "developer_transfers_created_by_users_id_fk": { + "name": "developer_transfers_created_by_users_id_fk", + "tableFrom": "developer_transfers", + "tableTo": "users", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "developer_transfers_accepted_by_users_id_fk": { + "name": "developer_transfers_accepted_by_users_id_fk", + "tableFrom": "developer_transfers", + "tableTo": "users", + "columnsFrom": ["accepted_by"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "developers": { + "name": "developers", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "owner_user_id": { + "name": "owner_user_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "approved_at": { + "name": "approved_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'1970-01-01T00:00:00.000Z'" + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'1970-01-01T00:00:00.000Z'" + }, + "avatar_url": { + "name": "avatar_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "contact_email": { + "name": "contact_email", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "ownership_epoch": { + "name": "ownership_epoch", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "content_revision": { + "name": "content_revision", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "profile_generation": { + "name": "profile_generation", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "approved_revision": { + "name": "approved_revision", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "approved_by": { + "name": "approved_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "github_org_verified": { + "name": "github_org_verified", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "github_verification_note": { + "name": "github_verification_note", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "github_verified_at": { + "name": "github_verified_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "github_url_verified": { + "name": "github_url_verified", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "url_check_cooldown_until": { + "name": "url_check_cooldown_until", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "idx_developers_identifier_nocase": { + "name": "idx_developers_identifier_nocase", + "columns": ["lower(\"identifier\")"], + "isUnique": true + }, + "idx_developers_owner_unique": { + "name": "idx_developers_owner_unique", + "columns": ["owner_user_id"], + "isUnique": true + }, + "idx_developers_approved": { + "name": "idx_developers_approved", + "columns": ["approved_at"], + "isUnique": false + } + }, + "foreignKeys": { + "developers_owner_user_id_users_id_fk": { + "name": "developers_owner_user_id_users_id_fk", + "tableFrom": "developers", + "tableTo": "users", + "columnsFrom": ["owner_user_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": { + "developers_ownership_epoch_check": { + "name": "developers_ownership_epoch_check", + "value": "\"developers\".\"ownership_epoch\" >= 1" + }, + "developers_content_revision_check": { + "name": "developers_content_revision_check", + "value": "\"developers\".\"content_revision\" >= 1" + }, + "developers_github_org_verified_check": { + "name": "developers_github_org_verified_check", + "value": "\"developers\".\"github_org_verified\" IN (0, 1)" + }, + "developers_github_url_verified_check": { + "name": "developers_github_url_verified_check", + "value": "\"developers\".\"github_url_verified\" = 1" + } + } + }, + "extension_resource_usage": { + "name": "extension_resource_usage", + "columns": { + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "subject": { + "name": "subject", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes": { + "name": "bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "extensions": { + "name": "extensions", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "revisions": { + "name": "revisions", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + } + }, + "indexes": { + "idx_extension_resource_usage_subject": { + "name": "idx_extension_resource_usage_subject", + "columns": ["scope", "subject"], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "extension_revisions": { + "name": "extension_revisions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "extension_id": { + "name": "extension_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "developer_id": { + "name": "developer_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "submitted_by": { + "name": "submitted_by", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content_bytes": { + "name": "content_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "summary_name": { + "name": "summary_name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "summary_version": { + "name": "summary_version", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "summary_description": { + "name": "summary_description", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "content_readable": { + "name": "content_readable", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "content_hash": { + "name": "content_hash", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "compacted_at": { + "name": "compacted_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reviewer_id": { + "name": "reviewer_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "review_note": { + "name": "review_note", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + }, + "reviewed_at": { + "name": "reviewed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "ownership_epoch": { + "name": "ownership_epoch", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + } + }, + "indexes": { + "idx_extension_revisions_retention": { + "name": "idx_extension_revisions_retention", + "columns": ["compacted_at", "reviewed_at", "id", "status"], + "isUnique": false + }, + "idx_extension_revisions_submitted_by": { + "name": "idx_extension_revisions_submitted_by", + "columns": ["submitted_by"], + "isUnique": false + }, + "idx_extension_revisions_developer": { + "name": "idx_extension_revisions_developer", + "columns": ["developer_id"], + "isUnique": false + }, + "idx_extension_revisions_pending": { + "name": "idx_extension_revisions_pending", + "columns": ["extension_id"], + "isUnique": true, + "where": "\"extension_revisions\".\"status\" = 'pending'" + }, + "idx_extension_revisions_extension_page": { + "name": "idx_extension_revisions_extension_page", + "columns": ["extension_id", "\"created_at\" desc", "\"id\" desc"], + "isUnique": false + }, + "idx_extension_revisions_submitter_page": { + "name": "idx_extension_revisions_submitter_page", + "columns": ["submitted_by", "\"created_at\" desc", "\"id\" desc"], + "isUnique": false + }, + "idx_extension_revisions_queue_page": { + "name": "idx_extension_revisions_queue_page", + "columns": ["status", "created_at", "id"], + "isUnique": false + }, + "idx_extension_revisions_reviewed": { + "name": "idx_extension_revisions_reviewed", + "columns": ["extension_id", "reviewed_at"], + "isUnique": false, + "where": "\"extension_revisions\".\"status\" IN ('approved', 'rejected')" + }, + "idx_extension_revisions_submitter_pending": { + "name": "idx_extension_revisions_submitter_pending", + "columns": ["submitted_by"], + "isUnique": false, + "where": "\"extension_revisions\".\"status\" = 'pending'" + } + }, + "foreignKeys": { + "extension_revisions_extension_id_extensions_id_fk": { + "name": "extension_revisions_extension_id_extensions_id_fk", + "tableFrom": "extension_revisions", + "tableTo": "extensions", + "columnsFrom": ["extension_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "extension_revisions_submitted_by_users_id_fk": { + "name": "extension_revisions_submitted_by_users_id_fk", + "tableFrom": "extension_revisions", + "tableTo": "users", + "columnsFrom": ["submitted_by"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "extension_revisions_reviewer_id_users_id_fk": { + "name": "extension_revisions_reviewer_id_users_id_fk", + "tableFrom": "extension_revisions", + "tableTo": "users", + "columnsFrom": ["reviewer_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": { + "extension_revisions_status_check": { + "name": "extension_revisions_status_check", + "value": "\"extension_revisions\".\"status\" IN ('pending', 'approved', 'rejected')" + }, + "extension_revisions_ownership_epoch_check": { + "name": "extension_revisions_ownership_epoch_check", + "value": "\"extension_revisions\".\"ownership_epoch\" >= 1" + } + } + }, + "extension_write_events": { + "name": "extension_write_events", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "developer_id": { + "name": "developer_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "occurred_at": { + "name": "occurred_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_extension_write_events_account": { + "name": "idx_extension_write_events_account", + "columns": ["account_id", "occurred_at"], + "isUnique": false + }, + "idx_extension_write_events_developer": { + "name": "idx_extension_write_events_developer", + "columns": ["developer_id", "occurred_at"], + "isUnique": false + }, + "idx_extension_write_events_time": { + "name": "idx_extension_write_events_time", + "columns": ["occurred_at"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "extensions": { + "name": "extensions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "developer_id": { + "name": "developer_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "published_bytes": { + "name": "published_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "published_at": { + "name": "published_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "published_revision_id": { + "name": "published_revision_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "manifest": { + "name": "manifest", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "releases": { + "name": "releases", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "website": { + "name": "website", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "license": { + "name": "license", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "icon_url": { + "name": "icon_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "readme": { + "name": "readme", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "version": { + "name": "version", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "download_url": { + "name": "download_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + }, + "delisted_at": { + "name": "delisted_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "delist_reason": { + "name": "delist_reason", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "idx_extensions_published_revision": { + "name": "idx_extensions_published_revision", + "columns": ["published_revision_id"], + "isUnique": false + }, + "idx_extensions_identifier_nocase": { + "name": "idx_extensions_identifier_nocase", + "columns": ["lower(\"identifier\")"], + "isUnique": true + }, + "idx_extensions_developer_order": { + "name": "idx_extensions_developer_order", + "columns": ["developer_id", "lower(\"identifier\")", "id"], + "isUnique": false + }, + "idx_extensions_public_order": { + "name": "idx_extensions_public_order", + "columns": ["lower(\"identifier\")", "id"], + "isUnique": false, + "where": "\"extensions\".\"published_at\" IS NOT NULL AND \"extensions\".\"delisted_at\" IS NULL" + }, + "idx_extensions_type_public_order": { + "name": "idx_extensions_type_public_order", + "columns": ["type", "lower(\"identifier\")", "id"], + "isUnique": false, + "where": "\"extensions\".\"published_at\" IS NOT NULL AND \"extensions\".\"delisted_at\" IS NULL" + } + }, + "foreignKeys": { + "extensions_developer_id_developers_id_fk": { + "name": "extensions_developer_id_developers_id_fk", + "tableFrom": "extensions", + "tableTo": "developers", + "columnsFrom": ["developer_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": { + "extensions_published_content_check": { + "name": "extensions_published_content_check", + "value": "\"extensions\".\"published_at\" IS NULL OR (\n \"extensions\".\"type\" IS NOT NULL AND \"extensions\".\"name\" IS NOT NULL AND\n \"extensions\".\"description\" IS NOT NULL AND \"extensions\".\"releases\" IS NOT NULL AND\n \"extensions\".\"website\" IS NOT NULL AND \"extensions\".\"license\" IS NOT NULL AND\n \"extensions\".\"readme\" IS NOT NULL AND \"extensions\".\"source\" IS NOT NULL AND\n \"extensions\".\"version\" IS NOT NULL AND \"extensions\".\"download_url\" IS NOT NULL\n )" + } + } + }, + "m2m_publisher_clients": { + "name": "m2m_publisher_clients", + "columns": { + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "developer_id": { + "name": "developer_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "can_publish": { + "name": "can_publish", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "ownership_epoch": { + "name": "ownership_epoch", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + }, + "revoked_at": { + "name": "revoked_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "idx_m2m_publisher_clients_developer": { + "name": "idx_m2m_publisher_clients_developer", + "columns": ["developer_id"], + "isUnique": false + } + }, + "foreignKeys": { + "m2m_publisher_clients_developer_id_developers_id_fk": { + "name": "m2m_publisher_clients_developer_id_developers_id_fk", + "tableFrom": "m2m_publisher_clients", + "tableTo": "developers", + "columnsFrom": ["developer_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": { + "m2m_publisher_clients_can_publish_check": { + "name": "m2m_publisher_clients_can_publish_check", + "value": "\"m2m_publisher_clients\".\"can_publish\" IN (0, 1)" + }, + "m2m_publisher_clients_ownership_epoch_check": { + "name": "m2m_publisher_clients_ownership_epoch_check", + "value": "\"m2m_publisher_clients\".\"ownership_epoch\" >= 1" + } + } + }, + "package_artifacts": { + "name": "package_artifacts", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "manifest": { + "name": "manifest", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "manifest_hash": { + "name": "manifest_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sha256": { + "name": "sha256", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "object_id": { + "name": "object_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "download_url": { + "name": "download_url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "published_at": { + "name": "published_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + } + }, + "indexes": { + "idx_package_artifact_manifest": { + "name": "idx_package_artifact_manifest", + "columns": ["source_id", "manifest_hash"], + "isUnique": true + }, + "idx_package_artifacts_object": { + "name": "idx_package_artifacts_object", + "columns": ["object_id"], + "isUnique": true + } + }, + "foreignKeys": { + "package_artifacts_source_id_package_sources_id_fk": { + "name": "package_artifacts_source_id_package_sources_id_fk", + "tableFrom": "package_artifacts", + "tableTo": "package_sources", + "columnsFrom": ["source_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "package_artifacts_object_id_package_objects_id_fk": { + "name": "package_artifacts_object_id_package_objects_id_fk", + "tableFrom": "package_artifacts", + "tableTo": "package_objects", + "columnsFrom": ["object_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "package_icons": { + "name": "package_icons", + "columns": { + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "avif_object_id": { + "name": "avif_object_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "avif_sha256": { + "name": "avif_sha256", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "png_object_id": { + "name": "png_object_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "png_sha256": { + "name": "png_sha256", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "svg_object_id": { + "name": "svg_object_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "svg_sha256": { + "name": "svg_sha256", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "package_icons_source_id_package_sources_id_fk": { + "name": "package_icons_source_id_package_sources_id_fk", + "tableFrom": "package_icons", + "tableTo": "package_sources", + "columnsFrom": ["source_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "package_icons_avif_object_id_package_objects_id_fk": { + "name": "package_icons_avif_object_id_package_objects_id_fk", + "tableFrom": "package_icons", + "tableTo": "package_objects", + "columnsFrom": ["avif_object_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "package_icons_png_object_id_package_objects_id_fk": { + "name": "package_icons_png_object_id_package_objects_id_fk", + "tableFrom": "package_icons", + "tableTo": "package_objects", + "columnsFrom": ["png_object_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "package_icons_svg_object_id_package_objects_id_fk": { + "name": "package_icons_svg_object_id_package_objects_id_fk", + "tableFrom": "package_icons", + "tableTo": "package_objects", + "columnsFrom": ["svg_object_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "package_objects": { + "name": "package_objects", + "columns": { + "deleting": { + "name": "deleting", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes": { + "name": "bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + } + }, + "indexes": { + "idx_package_objects_creator": { + "name": "idx_package_objects_creator", + "columns": ["created_by"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "package_sources": { + "name": "package_sources", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "profile_generation": { + "name": "profile_generation", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "extension_identifier": { + "name": "extension_identifier", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "version": { + "name": "version", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sha256": { + "name": "sha256", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "object_id": { + "name": "object_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + } + }, + "indexes": { + "idx_package_source_version": { + "name": "idx_package_source_version", + "columns": ["profile_generation", "extension_identifier", "version"], + "isUnique": true + }, + "idx_package_sources_extension": { + "name": "idx_package_sources_extension", + "columns": ["extension_identifier"], + "isUnique": false + }, + "idx_package_sources_object": { + "name": "idx_package_sources_object", + "columns": ["object_id"], + "isUnique": true + } + }, + "foreignKeys": { + "package_sources_object_id_package_objects_id_fk": { + "name": "package_sources_object_id_package_objects_id_fk", + "tableFrom": "package_sources", + "tableTo": "package_objects", + "columnsFrom": ["object_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "users": { + "name": "users", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "email_verified": { + "name": "email_verified", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "picture": { + "name": "picture", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "is_moderator": { + "name": "is_moderator", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "github_login": { + "name": "github_login", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "github_orgs": { + "name": "github_orgs", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "github_orgs_expires_at": { + "name": "github_orgs_expires_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": { + "idx_developers_identifier_nocase": { + "columns": { + "lower(\"identifier\")": { + "isExpression": true + } + } + }, + "idx_extension_revisions_extension_page": { + "columns": { + "\"created_at\" desc": { + "isExpression": true + }, + "\"id\" desc": { + "isExpression": true + } + } + }, + "idx_extension_revisions_submitter_page": { + "columns": { + "\"created_at\" desc": { + "isExpression": true + }, + "\"id\" desc": { + "isExpression": true + } + } + }, + "idx_extensions_identifier_nocase": { + "columns": { + "lower(\"identifier\")": { + "isExpression": true + } + } + }, + "idx_extensions_developer_order": { + "columns": { + "lower(\"identifier\")": { + "isExpression": true + } + } + }, + "idx_extensions_public_order": { + "columns": { + "lower(\"identifier\")": { + "isExpression": true + } + } + }, + "idx_extensions_type_public_order": { + "columns": { + "lower(\"identifier\")": { + "isExpression": true + } + } + } + } + } +} diff --git a/src/services/extensions/v2/db/migrations/meta/0032_snapshot.json b/src/services/extensions/v2/db/migrations/meta/0032_snapshot.json new file mode 100644 index 00000000..fb6cf6e5 --- /dev/null +++ b/src/services/extensions/v2/db/migrations/meta/0032_snapshot.json @@ -0,0 +1,1927 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "d557b6f4-8439-494b-b7e3-ef5e7eda15f8", + "prevId": "78430a63-8fd0-4d21-a557-33958d0386f8", + "tables": { + "claim_verification_budgets": { + "name": "claim_verification_budgets", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_claim_verification_budgets_expiry": { + "name": "idx_claim_verification_budgets_expiry", + "columns": ["expires_at"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "developer_claims": { + "name": "developer_claims", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "developer_id": { + "name": "developer_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "claimant_id": { + "name": "claimant_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "note": { + "name": "note", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "review_note": { + "name": "review_note", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reviewer_id": { + "name": "reviewer_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + }, + "reviewed_at": { + "name": "reviewed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "github_org_verified": { + "name": "github_org_verified", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "github_verification_note": { + "name": "github_verification_note", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "idx_developer_claims_developer": { + "name": "idx_developer_claims_developer", + "columns": ["developer_id"], + "isUnique": false + }, + "idx_developer_claims_claimant": { + "name": "idx_developer_claims_claimant", + "columns": ["claimant_id"], + "isUnique": false + }, + "idx_developer_claims_pending_unique": { + "name": "idx_developer_claims_pending_unique", + "columns": ["developer_id", "claimant_id"], + "isUnique": true, + "where": "\"developer_claims\".\"status\" = 'pending'" + }, + "idx_developer_claims_pending_queue": { + "name": "idx_developer_claims_pending_queue", + "columns": ["created_at"], + "isUnique": false, + "where": "\"developer_claims\".\"status\" = 'pending'" + } + }, + "foreignKeys": { + "developer_claims_developer_id_developers_id_fk": { + "name": "developer_claims_developer_id_developers_id_fk", + "tableFrom": "developer_claims", + "tableTo": "developers", + "columnsFrom": ["developer_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "developer_claims_claimant_id_users_id_fk": { + "name": "developer_claims_claimant_id_users_id_fk", + "tableFrom": "developer_claims", + "tableTo": "users", + "columnsFrom": ["claimant_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "developer_claims_reviewer_id_users_id_fk": { + "name": "developer_claims_reviewer_id_users_id_fk", + "tableFrom": "developer_claims", + "tableTo": "users", + "columnsFrom": ["reviewer_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": { + "developer_claims_status_check": { + "name": "developer_claims_status_check", + "value": "\"developer_claims\".\"status\" IN ('pending', 'approved', 'rejected')" + }, + "developer_claims_github_org_verified_check": { + "name": "developer_claims_github_org_verified_check", + "value": "\"developer_claims\".\"github_org_verified\" IN (0, 1)" + } + } + }, + "developer_history": { + "name": "developer_history", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "developer_id": { + "name": "developer_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "changed_by": { + "name": "changed_by", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "changed_at": { + "name": "changed_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + } + }, + "indexes": { + "idx_developer_history_developer_changed_at": { + "name": "idx_developer_history_developer_changed_at", + "columns": ["developer_id", "changed_at"], + "isUnique": false + }, + "idx_developer_history_account_changed_at": { + "name": "idx_developer_history_account_changed_at", + "columns": ["changed_by", "changed_at"], + "isUnique": false + } + }, + "foreignKeys": { + "developer_history_changed_by_users_id_fk": { + "name": "developer_history_changed_by_users_id_fk", + "tableFrom": "developer_history", + "tableTo": "users", + "columnsFrom": ["changed_by"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "developer_transfers": { + "name": "developer_transfers", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "developer_id": { + "name": "developer_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token_hash": { + "name": "token_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + }, + "expires_at": { + "name": "expires_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "accepted_by": { + "name": "accepted_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "accepted_at": { + "name": "accepted_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "revoked_at": { + "name": "revoked_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "idx_developer_transfers_token": { + "name": "idx_developer_transfers_token", + "columns": ["token_hash"], + "isUnique": true + }, + "idx_developer_transfers_pending": { + "name": "idx_developer_transfers_pending", + "columns": ["developer_id"], + "isUnique": true, + "where": "\"developer_transfers\".\"accepted_at\" IS NULL AND \"developer_transfers\".\"revoked_at\" IS NULL" + } + }, + "foreignKeys": { + "developer_transfers_developer_id_developers_id_fk": { + "name": "developer_transfers_developer_id_developers_id_fk", + "tableFrom": "developer_transfers", + "tableTo": "developers", + "columnsFrom": ["developer_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "developer_transfers_created_by_users_id_fk": { + "name": "developer_transfers_created_by_users_id_fk", + "tableFrom": "developer_transfers", + "tableTo": "users", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "developer_transfers_accepted_by_users_id_fk": { + "name": "developer_transfers_accepted_by_users_id_fk", + "tableFrom": "developer_transfers", + "tableTo": "users", + "columnsFrom": ["accepted_by"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "developers": { + "name": "developers", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "owner_user_id": { + "name": "owner_user_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "approved_at": { + "name": "approved_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'1970-01-01T00:00:00.000Z'" + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'1970-01-01T00:00:00.000Z'" + }, + "avatar_url": { + "name": "avatar_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "contact_email": { + "name": "contact_email", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "ownership_epoch": { + "name": "ownership_epoch", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "content_revision": { + "name": "content_revision", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "profile_generation": { + "name": "profile_generation", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "approved_revision": { + "name": "approved_revision", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "approved_by": { + "name": "approved_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "github_org_verified": { + "name": "github_org_verified", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "github_verification_note": { + "name": "github_verification_note", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "github_verified_at": { + "name": "github_verified_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "github_url_verified": { + "name": "github_url_verified", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "url_check_cooldown_until": { + "name": "url_check_cooldown_until", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "idx_developers_identifier_nocase": { + "name": "idx_developers_identifier_nocase", + "columns": ["lower(\"identifier\")"], + "isUnique": true + }, + "idx_developers_owner_unique": { + "name": "idx_developers_owner_unique", + "columns": ["owner_user_id"], + "isUnique": true + }, + "idx_developers_approved": { + "name": "idx_developers_approved", + "columns": ["approved_at"], + "isUnique": false + } + }, + "foreignKeys": { + "developers_owner_user_id_users_id_fk": { + "name": "developers_owner_user_id_users_id_fk", + "tableFrom": "developers", + "tableTo": "users", + "columnsFrom": ["owner_user_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": { + "developers_ownership_epoch_check": { + "name": "developers_ownership_epoch_check", + "value": "\"developers\".\"ownership_epoch\" >= 1" + }, + "developers_content_revision_check": { + "name": "developers_content_revision_check", + "value": "\"developers\".\"content_revision\" >= 1" + }, + "developers_github_org_verified_check": { + "name": "developers_github_org_verified_check", + "value": "\"developers\".\"github_org_verified\" IN (0, 1)" + }, + "developers_github_url_verified_check": { + "name": "developers_github_url_verified_check", + "value": "\"developers\".\"github_url_verified\" = 1" + } + } + }, + "extension_resource_usage": { + "name": "extension_resource_usage", + "columns": { + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "subject": { + "name": "subject", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes": { + "name": "bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "extensions": { + "name": "extensions", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "revisions": { + "name": "revisions", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + } + }, + "indexes": { + "idx_extension_resource_usage_subject": { + "name": "idx_extension_resource_usage_subject", + "columns": ["scope", "subject"], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "extension_revisions": { + "name": "extension_revisions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "extension_id": { + "name": "extension_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "developer_id": { + "name": "developer_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "submitted_by": { + "name": "submitted_by", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content_bytes": { + "name": "content_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "summary_name": { + "name": "summary_name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "summary_version": { + "name": "summary_version", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "summary_description": { + "name": "summary_description", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "content_readable": { + "name": "content_readable", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "content_hash": { + "name": "content_hash", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "compacted_at": { + "name": "compacted_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reviewer_id": { + "name": "reviewer_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "review_note": { + "name": "review_note", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + }, + "reviewed_at": { + "name": "reviewed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "ownership_epoch": { + "name": "ownership_epoch", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + } + }, + "indexes": { + "idx_extension_revisions_retention": { + "name": "idx_extension_revisions_retention", + "columns": ["compacted_at", "reviewed_at", "id", "status"], + "isUnique": false + }, + "idx_extension_revisions_submitted_by": { + "name": "idx_extension_revisions_submitted_by", + "columns": ["submitted_by"], + "isUnique": false + }, + "idx_extension_revisions_developer": { + "name": "idx_extension_revisions_developer", + "columns": ["developer_id"], + "isUnique": false + }, + "idx_extension_revisions_pending": { + "name": "idx_extension_revisions_pending", + "columns": ["extension_id"], + "isUnique": true, + "where": "\"extension_revisions\".\"status\" = 'pending'" + }, + "idx_extension_revisions_extension_page": { + "name": "idx_extension_revisions_extension_page", + "columns": ["extension_id", "\"created_at\" desc", "\"id\" desc"], + "isUnique": false + }, + "idx_extension_revisions_submitter_page": { + "name": "idx_extension_revisions_submitter_page", + "columns": ["submitted_by", "\"created_at\" desc", "\"id\" desc"], + "isUnique": false + }, + "idx_extension_revisions_queue_page": { + "name": "idx_extension_revisions_queue_page", + "columns": ["status", "created_at", "id"], + "isUnique": false + }, + "idx_extension_revisions_reviewed": { + "name": "idx_extension_revisions_reviewed", + "columns": ["extension_id", "reviewed_at"], + "isUnique": false, + "where": "\"extension_revisions\".\"status\" IN ('approved', 'rejected')" + }, + "idx_extension_revisions_submitter_pending": { + "name": "idx_extension_revisions_submitter_pending", + "columns": ["submitted_by"], + "isUnique": false, + "where": "\"extension_revisions\".\"status\" = 'pending'" + } + }, + "foreignKeys": { + "extension_revisions_extension_id_extensions_id_fk": { + "name": "extension_revisions_extension_id_extensions_id_fk", + "tableFrom": "extension_revisions", + "tableTo": "extensions", + "columnsFrom": ["extension_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "extension_revisions_submitted_by_users_id_fk": { + "name": "extension_revisions_submitted_by_users_id_fk", + "tableFrom": "extension_revisions", + "tableTo": "users", + "columnsFrom": ["submitted_by"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "extension_revisions_reviewer_id_users_id_fk": { + "name": "extension_revisions_reviewer_id_users_id_fk", + "tableFrom": "extension_revisions", + "tableTo": "users", + "columnsFrom": ["reviewer_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": { + "extension_revisions_status_check": { + "name": "extension_revisions_status_check", + "value": "\"extension_revisions\".\"status\" IN ('pending', 'approved', 'rejected')" + }, + "extension_revisions_ownership_epoch_check": { + "name": "extension_revisions_ownership_epoch_check", + "value": "\"extension_revisions\".\"ownership_epoch\" >= 1" + } + } + }, + "extension_write_events": { + "name": "extension_write_events", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "developer_id": { + "name": "developer_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "occurred_at": { + "name": "occurred_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_extension_write_events_account": { + "name": "idx_extension_write_events_account", + "columns": ["account_id", "occurred_at"], + "isUnique": false + }, + "idx_extension_write_events_developer": { + "name": "idx_extension_write_events_developer", + "columns": ["developer_id", "occurred_at"], + "isUnique": false + }, + "idx_extension_write_events_time": { + "name": "idx_extension_write_events_time", + "columns": ["occurred_at"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "extensions": { + "name": "extensions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "developer_id": { + "name": "developer_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "published_bytes": { + "name": "published_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "published_at": { + "name": "published_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "published_revision_id": { + "name": "published_revision_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "manifest": { + "name": "manifest", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "releases": { + "name": "releases", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "website": { + "name": "website", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "license": { + "name": "license", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "icon_url": { + "name": "icon_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "readme": { + "name": "readme", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "version": { + "name": "version", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "download_url": { + "name": "download_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + }, + "delisted_at": { + "name": "delisted_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "delist_reason": { + "name": "delist_reason", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "idx_extensions_published_revision": { + "name": "idx_extensions_published_revision", + "columns": ["published_revision_id"], + "isUnique": false + }, + "idx_extensions_identifier_nocase": { + "name": "idx_extensions_identifier_nocase", + "columns": ["lower(\"identifier\")"], + "isUnique": true + }, + "idx_extensions_developer_order": { + "name": "idx_extensions_developer_order", + "columns": ["developer_id", "lower(\"identifier\")", "id"], + "isUnique": false + }, + "idx_extensions_public_order": { + "name": "idx_extensions_public_order", + "columns": ["lower(\"identifier\")", "id"], + "isUnique": false, + "where": "\"extensions\".\"published_at\" IS NOT NULL AND \"extensions\".\"delisted_at\" IS NULL" + }, + "idx_extensions_type_public_order": { + "name": "idx_extensions_type_public_order", + "columns": ["type", "lower(\"identifier\")", "id"], + "isUnique": false, + "where": "\"extensions\".\"published_at\" IS NOT NULL AND \"extensions\".\"delisted_at\" IS NULL" + } + }, + "foreignKeys": { + "extensions_developer_id_developers_id_fk": { + "name": "extensions_developer_id_developers_id_fk", + "tableFrom": "extensions", + "tableTo": "developers", + "columnsFrom": ["developer_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": { + "extensions_published_content_check": { + "name": "extensions_published_content_check", + "value": "\"extensions\".\"published_at\" IS NULL OR (\n \"extensions\".\"type\" IS NOT NULL AND \"extensions\".\"name\" IS NOT NULL AND\n \"extensions\".\"description\" IS NOT NULL AND \"extensions\".\"releases\" IS NOT NULL AND\n \"extensions\".\"website\" IS NOT NULL AND \"extensions\".\"license\" IS NOT NULL AND\n \"extensions\".\"readme\" IS NOT NULL AND \"extensions\".\"source\" IS NOT NULL AND\n \"extensions\".\"version\" IS NOT NULL AND \"extensions\".\"download_url\" IS NOT NULL\n )" + } + } + }, + "m2m_publisher_clients": { + "name": "m2m_publisher_clients", + "columns": { + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "developer_id": { + "name": "developer_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "can_publish": { + "name": "can_publish", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "ownership_epoch": { + "name": "ownership_epoch", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + }, + "revoked_at": { + "name": "revoked_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "idx_m2m_publisher_clients_developer": { + "name": "idx_m2m_publisher_clients_developer", + "columns": ["developer_id"], + "isUnique": false + } + }, + "foreignKeys": { + "m2m_publisher_clients_developer_id_developers_id_fk": { + "name": "m2m_publisher_clients_developer_id_developers_id_fk", + "tableFrom": "m2m_publisher_clients", + "tableTo": "developers", + "columnsFrom": ["developer_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": { + "m2m_publisher_clients_can_publish_check": { + "name": "m2m_publisher_clients_can_publish_check", + "value": "\"m2m_publisher_clients\".\"can_publish\" IN (0, 1)" + }, + "m2m_publisher_clients_ownership_epoch_check": { + "name": "m2m_publisher_clients_ownership_epoch_check", + "value": "\"m2m_publisher_clients\".\"ownership_epoch\" >= 1" + } + } + }, + "package_artifacts": { + "name": "package_artifacts", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "manifest": { + "name": "manifest", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "manifest_hash": { + "name": "manifest_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sha256": { + "name": "sha256", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "object_id": { + "name": "object_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "download_url": { + "name": "download_url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "published_at": { + "name": "published_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + } + }, + "indexes": { + "idx_package_artifact_manifest": { + "name": "idx_package_artifact_manifest", + "columns": ["source_id", "manifest_hash"], + "isUnique": true + }, + "idx_package_artifacts_object": { + "name": "idx_package_artifacts_object", + "columns": ["object_id"], + "isUnique": true + } + }, + "foreignKeys": { + "package_artifacts_source_id_package_sources_id_fk": { + "name": "package_artifacts_source_id_package_sources_id_fk", + "tableFrom": "package_artifacts", + "tableTo": "package_sources", + "columnsFrom": ["source_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "package_artifacts_object_id_package_objects_id_fk": { + "name": "package_artifacts_object_id_package_objects_id_fk", + "tableFrom": "package_artifacts", + "tableTo": "package_objects", + "columnsFrom": ["object_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "package_icons": { + "name": "package_icons", + "columns": { + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "avif_object_id": { + "name": "avif_object_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "avif_sha256": { + "name": "avif_sha256", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "png_object_id": { + "name": "png_object_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "png_sha256": { + "name": "png_sha256", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "svg_object_id": { + "name": "svg_object_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "svg_sha256": { + "name": "svg_sha256", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "package_icons_source_id_package_sources_id_fk": { + "name": "package_icons_source_id_package_sources_id_fk", + "tableFrom": "package_icons", + "tableTo": "package_sources", + "columnsFrom": ["source_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "package_icons_avif_object_id_package_objects_id_fk": { + "name": "package_icons_avif_object_id_package_objects_id_fk", + "tableFrom": "package_icons", + "tableTo": "package_objects", + "columnsFrom": ["avif_object_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "package_icons_png_object_id_package_objects_id_fk": { + "name": "package_icons_png_object_id_package_objects_id_fk", + "tableFrom": "package_icons", + "tableTo": "package_objects", + "columnsFrom": ["png_object_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "package_icons_svg_object_id_package_objects_id_fk": { + "name": "package_icons_svg_object_id_package_objects_id_fk", + "tableFrom": "package_icons", + "tableTo": "package_objects", + "columnsFrom": ["svg_object_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "package_jobs": { + "name": "package_jobs", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "fingerprint": { + "name": "fingerprint", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "owner_user_id": { + "name": "owner_user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "owner": { + "name": "owner", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "manifest": { + "name": "manifest", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "upload_object_id": { + "name": "upload_object_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "upload_sha256": { + "name": "upload_sha256", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'queued'" + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "lease_id": { + "name": "lease_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "lease_until": { + "name": "lease_until", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "artifact_id": { + "name": "artifact_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error_code": { + "name": "error_code", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error_status": { + "name": "error_status", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "enqueued_at": { + "name": "enqueued_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + } + }, + "indexes": { + "idx_package_jobs_fingerprint": { + "name": "idx_package_jobs_fingerprint", + "columns": ["fingerprint"], + "isUnique": true, + "where": "\"package_jobs\".\"status\" != 'failed'" + }, + "idx_package_jobs_owner": { + "name": "idx_package_jobs_owner", + "columns": ["owner_user_id"], + "isUnique": false + }, + "idx_package_jobs_upload_object": { + "name": "idx_package_jobs_upload_object", + "columns": ["upload_object_id"], + "isUnique": false + }, + "idx_package_jobs_dispatch": { + "name": "idx_package_jobs_dispatch", + "columns": ["status", "enqueued_at"], + "isUnique": false + } + }, + "foreignKeys": { + "package_jobs_upload_object_id_package_objects_id_fk": { + "name": "package_jobs_upload_object_id_package_objects_id_fk", + "tableFrom": "package_jobs", + "tableTo": "package_objects", + "columnsFrom": ["upload_object_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "package_jobs_artifact_id_package_artifacts_id_fk": { + "name": "package_jobs_artifact_id_package_artifacts_id_fk", + "tableFrom": "package_jobs", + "tableTo": "package_artifacts", + "columnsFrom": ["artifact_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": { + "package_jobs_status_check": { + "name": "package_jobs_status_check", + "value": "\"package_jobs\".\"status\" IN ('queued','processing','ready','failed')" + } + } + }, + "package_objects": { + "name": "package_objects", + "columns": { + "deleting": { + "name": "deleting", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes": { + "name": "bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + } + }, + "indexes": { + "idx_package_objects_creator": { + "name": "idx_package_objects_creator", + "columns": ["created_by"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "package_sources": { + "name": "package_sources", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "profile_generation": { + "name": "profile_generation", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "extension_identifier": { + "name": "extension_identifier", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "version": { + "name": "version", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sha256": { + "name": "sha256", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "object_id": { + "name": "object_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + } + }, + "indexes": { + "idx_package_source_version": { + "name": "idx_package_source_version", + "columns": ["profile_generation", "extension_identifier", "version"], + "isUnique": true + }, + "idx_package_sources_extension": { + "name": "idx_package_sources_extension", + "columns": ["extension_identifier"], + "isUnique": false + }, + "idx_package_sources_object": { + "name": "idx_package_sources_object", + "columns": ["object_id"], + "isUnique": true + } + }, + "foreignKeys": { + "package_sources_object_id_package_objects_id_fk": { + "name": "package_sources_object_id_package_objects_id_fk", + "tableFrom": "package_sources", + "tableTo": "package_objects", + "columnsFrom": ["object_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "users": { + "name": "users", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "email_verified": { + "name": "email_verified", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "picture": { + "name": "picture", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "is_moderator": { + "name": "is_moderator", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "github_login": { + "name": "github_login", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "github_orgs": { + "name": "github_orgs", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "github_orgs_expires_at": { + "name": "github_orgs_expires_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": { + "idx_developers_identifier_nocase": { + "columns": { + "lower(\"identifier\")": { + "isExpression": true + } + } + }, + "idx_extension_revisions_extension_page": { + "columns": { + "\"created_at\" desc": { + "isExpression": true + }, + "\"id\" desc": { + "isExpression": true + } + } + }, + "idx_extension_revisions_submitter_page": { + "columns": { + "\"created_at\" desc": { + "isExpression": true + }, + "\"id\" desc": { + "isExpression": true + } + } + }, + "idx_extensions_identifier_nocase": { + "columns": { + "lower(\"identifier\")": { + "isExpression": true + } + } + }, + "idx_extensions_developer_order": { + "columns": { + "lower(\"identifier\")": { + "isExpression": true + } + } + }, + "idx_extensions_public_order": { + "columns": { + "lower(\"identifier\")": { + "isExpression": true + } + } + }, + "idx_extensions_type_public_order": { + "columns": { + "lower(\"identifier\")": { + "isExpression": true + } + } + } + } + } +} diff --git a/src/services/extensions/v2/db/migrations/meta/_journal.json b/src/services/extensions/v2/db/migrations/meta/_journal.json index 651ad1a8..122ae010 100644 --- a/src/services/extensions/v2/db/migrations/meta/_journal.json +++ b/src/services/extensions/v2/db/migrations/meta/_journal.json @@ -120,6 +120,20 @@ "when": 1794071864002, "tag": "0030_directory_identity", "breakpoints": true + }, + { + "idx": 31, + "version": "6", + "when": 1794078741225, + "tag": "0031_directory_packages", + "breakpoints": true + }, + { + "idx": 32, + "version": "6", + "when": 1794078779548, + "tag": "0032_package_jobs", + "breakpoints": true } ] } diff --git a/src/services/extensions/v2/db/package-builds.ts b/src/services/extensions/v2/db/package-builds.ts new file mode 100644 index 00000000..6881017b --- /dev/null +++ b/src/services/extensions/v2/db/package-builds.ts @@ -0,0 +1,330 @@ +import { v7 as uuidv7 } from "uuid"; +import { ARTIFACT_SELECT } from "./packages"; +import { canonicalJson } from "../packages/manifest"; +import type { Manifest } from "../schemas/manifest"; +import type { + PackageRepository, + PackageOwner, + PackagePublisher, + SourceRow, + ArtifactRow, + IconObject +} from "../packages/repository"; +import { PackageError } from "../packages/errors"; + +export const MAX_PACKAGE_STORAGE_BYTES = 512 * 1024 * 1024; +export const MAX_PACKAGE_OBJECTS = 256; + +function ownerPredicate() { + return `EXISTS (SELECT 1 FROM developers d JOIN users u ON u.id=d.owner_user_id WHERE d.id=? AND d.profile_generation=? AND d.ownership_epoch=? AND d.owner_user_id=? AND u.deleted_at IS NULL AND NOT EXISTS (SELECT 1 FROM extensions e WHERE lower(e.identifier)=lower(?) AND e.developer_id!=d.id) AND (? IS NULL OR EXISTS (SELECT 1 FROM m2m_publisher_clients p WHERE p.client_id=? AND p.developer_id=d.id AND p.ownership_epoch=d.ownership_epoch AND p.revoked_at IS NULL)))`; +} +function ownerParams(owner: PackageOwner, id: string) { + return [ + owner.id, + owner.profile_generation, + owner.ownership_epoch, + owner.owner_user_id, + id, + owner.publisher_client_id ?? null, + owner.publisher_client_id ?? null + ]; +} + +export class PackageBuildDatabase implements PackageRepository { + constructor(private readonly db: D1Database) {} + async getOwner( + userId: string, + extensionId: string, + publisher?: PackagePublisher | null + ) { + const owner = await this.db + .prepare( + `SELECT d.id,d.profile_generation,d.ownership_epoch,d.owner_user_id FROM developers d JOIN users u ON u.id=d.owner_user_id WHERE (? IS NULL AND d.owner_user_id=? OR d.id=? AND d.ownership_epoch=? AND EXISTS (SELECT 1 FROM m2m_publisher_clients p WHERE p.client_id=? AND p.developer_id=d.id AND p.ownership_epoch=d.ownership_epoch AND p.revoked_at IS NULL)) AND u.deleted_at IS NULL AND NOT EXISTS (SELECT 1 FROM extensions e WHERE lower(e.identifier)=lower(?) AND e.developer_id!=d.id)` + ) + .bind( + publisher?.clientId ?? null, + userId, + publisher?.developerId ?? null, + publisher?.ownershipEpoch ?? null, + publisher?.clientId ?? null, + extensionId + ) + .first(); + if (!owner) + throw new PackageError( + "FORBIDDEN", + "An active owning developer profile is required", + 403 + ); + return { ...owner, publisher_client_id: publisher?.clientId }; + } + async findSource(generation: string, identifier: string, version: string) { + return this.db + .prepare( + "SELECT id,sha256,object_id FROM package_sources WHERE profile_generation=? AND extension_identifier=? AND version=?" + ) + .bind(generation, identifier, version) + .first(); + } + async findArtifact(sourceId: string, manifestHash: string) { + return this.db + .prepare(`${ARTIFACT_SELECT} WHERE a.source_id=? AND a.manifest_hash=?`) + .bind(sourceId, manifestHash) + .first(); + } + async hasIcon(sourceId: string) { + return Boolean( + await this.db + .prepare("SELECT 1 FROM package_icons WHERE source_id=?") + .bind(sourceId) + .first() + ); + } + async reserveObject( + owner: PackageOwner, + extensionId: string, + id: string, + bytes: number + ) { + const result = await this.db + .prepare( + `INSERT INTO package_objects (id,created_by,bytes) + SELECT ?,?,? WHERE ${ownerPredicate()} + AND (SELECT count(*) FROM package_objects WHERE created_by=?) < ? + AND (SELECT coalesce(sum(bytes),0) FROM package_objects WHERE created_by=?) + ? <= ?` + ) + .bind( + id, + owner.owner_user_id, + bytes, + ...ownerParams(owner, extensionId), + owner.owner_user_id, + MAX_PACKAGE_OBJECTS, + owner.owner_user_id, + bytes, + MAX_PACKAGE_STORAGE_BYTES + ) + .run(); + if (!result.meta.changes) + throw new PackageError( + "PACKAGE_ADMISSION", + "Ownership changed or package storage quota is exhausted", + 409 + ); + } + async removeObject(id: string) { + await this.db + .prepare("DELETE FROM package_objects WHERE id=?") + .bind(id) + .run(); + } + async claimCleanup() { + const result = await this.db + .prepare( + `UPDATE package_objects SET deleting=1 WHERE id IN ( + SELECT o.id FROM package_objects o WHERE (o.deleting=1 OR o.created_at < datetime('now','-1 day')) + AND NOT EXISTS (SELECT 1 FROM package_jobs j WHERE j.upload_object_id=o.id AND j.status IN ('queued','processing')) + AND NOT EXISTS (SELECT 1 FROM package_sources s WHERE s.object_id=o.id) + AND NOT EXISTS (SELECT 1 FROM package_artifacts a WHERE a.object_id=o.id) + AND NOT EXISTS (SELECT 1 FROM package_icons i WHERE i.avif_object_id=o.id OR i.png_object_id=o.id OR i.svg_object_id=o.id) LIMIT 100 + ) RETURNING id` + ) + .all<{ id: string }>(); + return result.results.map((row) => row.id); + } + async finishCleanup(id: string) { + await this.db + .prepare( + "UPDATE package_jobs SET upload_object_id=NULL WHERE upload_object_id=? AND status IN ('ready','failed')" + ) + .bind(id) + .run(); + await this.db + .prepare("DELETE FROM package_objects WHERE id=? AND deleting=1") + .bind(id) + .run(); + } + async linkSource(owner: PackageOwner, manifest: Manifest, source: SourceRow) { + await this.db + .prepare( + `INSERT INTO package_sources (id,profile_generation,extension_identifier,version,sha256,object_id) + SELECT ?,?,?,?,?,? WHERE ${ownerPredicate()} AND EXISTS (SELECT 1 FROM package_objects WHERE id=? AND deleting=0) + ON CONFLICT DO NOTHING` + ) + .bind( + source.id, + owner.profile_generation, + manifest.id, + manifest.version, + source.sha256, + source.object_id, + ...ownerParams(owner, manifest.id), + source.object_id + ) + .run(); + return await this.db + .prepare( + "SELECT id,sha256,object_id FROM package_sources WHERE profile_generation=? AND extension_identifier=? AND version=?" + ) + .bind(owner.profile_generation, manifest.id, manifest.version) + .first(); + } + async ensureDraft(owner: PackageOwner, identifier: string) { + // Reserve the permanent identity only after ZIP and icon validation succeeds. + await this.db + .prepare( + `INSERT INTO extensions (id,identifier,developer_id,created_by) + SELECT ?,?,?,? WHERE ${ownerPredicate()} ON CONFLICT DO NOTHING` + ) + .bind( + uuidv7(), + identifier, + owner.id, + owner.owner_user_id, + ...ownerParams(owner, identifier) + ) + .run(); + const extension = await this.db + .prepare( + "SELECT id FROM extensions WHERE lower(identifier)=lower(?) AND developer_id=?" + ) + .bind(identifier, owner.id) + .first<{ id: string }>(); + if (!extension) + throw new PackageError( + "FORBIDDEN", + "Extension ownership changed while preparing the package", + 403 + ); + return extension.id; + } + async linkArtifact( + owner: PackageOwner, + manifest: Manifest, + artifact: { + id: string; + sourceId: string; + manifestHash: string; + sha256: string; + objectId: string; + downloadUrl: string; + } + ) { + await this.db + .prepare( + `INSERT INTO package_artifacts (id,source_id,manifest,manifest_hash,sha256,object_id,download_url) + SELECT ?,?,?,?,?,?,? WHERE ${ownerPredicate()} AND EXISTS (SELECT 1 FROM package_objects WHERE id=? AND deleting=0) + ON CONFLICT DO NOTHING` + ) + .bind( + artifact.id, + artifact.sourceId, + canonicalJson(manifest), + artifact.manifestHash, + artifact.sha256, + artifact.objectId, + artifact.downloadUrl, + ...ownerParams(owner, manifest.id), + artifact.objectId + ) + .run(); + return this.findArtifact(artifact.sourceId, artifact.manifestHash); + } + async linkIcon( + owner: PackageOwner, + identifier: string, + sourceId: string, + objects: IconObject[] + ) { + const avif = objects.find((object) => object.format === "avif"); + const png = objects.find((object) => object.format === "png"); + const svg = objects.find((object) => object.format === "svg"); + const result = await this.db + .prepare( + `INSERT INTO package_icons (source_id,avif_object_id,avif_sha256,png_object_id,png_sha256,svg_object_id,svg_sha256) + SELECT ?,?,?,?,?,?,? WHERE ${ownerPredicate()} + AND NOT EXISTS (SELECT 1 FROM package_objects WHERE id IN (?,?,?) AND deleting=1) + ON CONFLICT DO NOTHING` + ) + .bind( + sourceId, + avif?.id ?? null, + avif?.checksum ?? null, + png?.id ?? null, + png?.checksum ?? null, + svg?.id ?? null, + svg?.checksum ?? null, + ...ownerParams(owner, identifier), + avif?.id ?? null, + png?.id ?? null, + svg?.id ?? null + ) + .run(); + return Boolean(result.meta.changes); + } + async getArtifact(id: string) { + return this.db + .prepare(`${ARTIFACT_SELECT} WHERE a.id=?`) + .bind(id) + .first(); + } + async isModerator(userId: string) { + return Boolean( + await this.db + .prepare( + "SELECT 1 FROM users WHERE id=? AND is_moderator=1 AND deleted_at IS NULL" + ) + .bind(userId) + .first() + ); + } + async correctionOwner(userId: string, sourceId: string, manifest: Manifest) { + return this.db + .prepare( + `SELECT d.owner_user_id FROM package_sources s + JOIN developers d ON d.profile_generation=s.profile_generation + JOIN users moderator ON moderator.id=? AND moderator.is_moderator=1 AND moderator.deleted_at IS NULL + WHERE s.id=? AND s.extension_identifier=? AND s.version=? + AND EXISTS (SELECT 1 FROM package_artifacts a WHERE a.source_id=s.id AND a.published_at IS NOT NULL)` + ) + .bind(userId, sourceId, manifest.id, manifest.version) + .first<{ owner_user_id: string }>(); + } + async getArtifactForReader(id: string, userId: string) { + return this.db + .prepare( + `${ARTIFACT_SELECT} WHERE a.id=? AND ( + (a.published_at IS NOT NULL AND EXISTS (SELECT 1 FROM extensions e WHERE lower(e.identifier)=lower(s.extension_identifier) AND e.published_at IS NOT NULL AND e.delisted_at IS NULL)) + OR EXISTS (SELECT 1 FROM users u WHERE u.id=? AND u.deleted_at IS NULL AND (u.is_moderator=1 OR EXISTS (SELECT 1 FROM developers d WHERE d.profile_generation=s.profile_generation AND d.owner_user_id=u.id))) + )` + ) + .bind(id, userId) + .first(); + } + async publishedIcon(identifier: string) { + return this.db + .prepare( + `SELECT e.published_revision_id, coalesce(i.svg_object_id,i.png_object_id) AS object_key + FROM extensions e + JOIN package_artifacts a ON a.download_url=e.download_url AND a.published_at IS NOT NULL + JOIN package_sources s ON s.id=a.source_id AND s.extension_identifier=lower(e.identifier) + JOIN package_icons i ON i.source_id=s.id + WHERE lower(e.identifier)=? AND e.published_at IS NOT NULL AND e.delisted_at IS NULL` + ) + .bind(identifier) + .first<{ published_revision_id: string; object_key: string | null }>(); + } + async updatePublishedIcon( + identifier: string, + revisionId: string, + url: string + ) { + const result = await this.db + .prepare( + "UPDATE extensions SET icon_url=? WHERE lower(identifier)=? AND published_revision_id=? AND delisted_at IS NULL AND icon_url IS NOT ?" + ) + .bind(url, identifier, revisionId, url) + .run(); + return Boolean(result.meta.changes); + } +} diff --git a/src/services/extensions/v2/db/package-jobs.ts b/src/services/extensions/v2/db/package-jobs.ts new file mode 100644 index 00000000..0a943ac9 --- /dev/null +++ b/src/services/extensions/v2/db/package-jobs.ts @@ -0,0 +1,169 @@ +import type { PackageJob, PackageJobRepository } from "../packages/jobs"; +import { PackageError } from "../packages/errors"; + +interface JobRow { + id: string; + fingerprint: string; + owner: string; + manifest: string; + source_id: string | null; + upload_object_id: string | null; + upload_sha256: string | null; + status: PackageJob["status"]; + attempts: number; + artifact_id: string | null; + error_code: string | null; + error_message: string | null; + error_status: number | null; + created_at: string; + updated_at: string; +} +const job = (row: JobRow): PackageJob => ({ + id: row.id, + fingerprint: row.fingerprint, + owner: JSON.parse(row.owner), + manifest: JSON.parse(row.manifest), + sourceId: row.source_id, + uploadObjectId: row.upload_object_id, + uploadSha256: row.upload_sha256, + status: row.status, + attempts: row.attempts, + artifactId: row.artifact_id, + error: row.error_code + ? { + code: row.error_code, + message: row.error_message!, + status: row.error_status! + } + : null, + createdAt: row.created_at, + updatedAt: row.updated_at +}); + +export class PackageJobsDatabase implements PackageJobRepository { + constructor(private readonly db: D1Database) {} + + async get(id: string) { + const row = await this.db + .prepare("SELECT * FROM package_jobs WHERE id=?") + .bind(id) + .first(); + return row ? job(row) : null; + } + async find(fingerprint: string) { + const row = await this.db + .prepare( + "SELECT * FROM package_jobs WHERE fingerprint=? AND status!='failed'" + ) + .bind(fingerprint) + .first(); + return row ? job(row) : null; + } + async create( + input: Pick< + PackageJob, + | "fingerprint" + | "owner" + | "manifest" + | "sourceId" + | "uploadObjectId" + | "uploadSha256" + > + ) { + const id = crypto.randomUUID(); + await this.db + .prepare( + `INSERT INTO package_jobs (id,fingerprint,owner_user_id,owner,manifest,source_id,upload_object_id,upload_sha256) + SELECT ?,?,?,?,?,?,?,? WHERE (SELECT count(*) FROM package_jobs WHERE owner_user_id=?) < 256 + ON CONFLICT DO NOTHING` + ) + .bind( + id, + input.fingerprint, + input.owner.owner_user_id, + JSON.stringify(input.owner), + JSON.stringify(input.manifest), + input.sourceId, + input.uploadObjectId, + input.uploadSha256, + input.owner.owner_user_id + ) + .run(); + const result = await this.find(input.fingerprint); + if (!result) + throw new PackageError( + "PACKAGE_ADMISSION", + "Package job allowance is exhausted; try again later", + 429 + ); + return result; + } + async claim(id: string, lease: string) { + const row = await this.db + .prepare( + `UPDATE package_jobs SET status='processing',attempts=attempts+1,lease_id=?,lease_until=datetime('now','+20 minutes'),updated_at=CURRENT_TIMESTAMP + WHERE id=? AND (status='queued' OR status='processing' AND lease_until(); + return row ? job(row) : null; + } + async complete(id: string, lease: string, artifactId: string) { + await this.db + .prepare( + "UPDATE package_jobs SET status='ready',artifact_id=?,error_code=NULL,error_message=NULL,error_status=NULL,lease_id=NULL,lease_until=NULL,updated_at=CURRENT_TIMESTAMP WHERE id=? AND lease_id=?" + ) + .bind(artifactId, id, lease) + .run(); + } + async fail( + id: string, + lease: string, + error: NonNullable, + retry: boolean + ) { + await this.db + .prepare( + "UPDATE package_jobs SET status=?,error_code=?,error_message=?,error_status=?,lease_id=NULL,lease_until=NULL,updated_at=CURRENT_TIMESTAMP WHERE id=? AND lease_id=?" + ) + .bind( + retry ? "queued" : "failed", + error.code, + error.message, + error.status, + id, + lease + ) + .run(); + } + async pending() { + const result = await this.db + .prepare( + `SELECT id FROM package_jobs WHERE + (status='queued' AND (enqueued_at IS NULL OR enqueued_at(); + return result.results.map((row) => row.id); + } + async markEnqueued(id: string) { + await this.db + .prepare( + "UPDATE package_jobs SET enqueued_at=CURRENT_TIMESTAMP WHERE id=?" + ) + .bind(id) + .run(); + } + async expire() { + await this.db + .prepare( + `UPDATE package_jobs SET status='failed',error_code='PACKAGE_EXPIRED',error_message='Package processing expired. Please try again.',error_status=503,lease_id=NULL,lease_until=NULL,updated_at=CURRENT_TIMESTAMP WHERE status IN ('queued','processing') AND created_at { + id = id.toLowerCase(); + if (!developerId) { + const ref = await db + .prepare( + `SELECT d.id FROM extensions e JOIN developers d ON d.id=e.developer_id WHERE lower(e.identifier)=lower(?) AND (? IS NULL OR d.owner_user_id=?)` + ) + .bind(id, callerId ?? null, callerId ?? null) + .first<{ id: string }>(); + if (!ref) return null; + developerId = ref.id; + } + const published = await db + .prepare( + `SELECT DISTINCT s.version FROM package_artifacts a JOIN package_sources s ON s.id=a.source_id WHERE s.extension_identifier=? AND a.published_at IS NOT NULL` + ) + .bind(id) + .all<{ version: string }>(); + const publishedVersions = new Set( + published.results.map((row) => row.version) + ); + const managed = publishedVersions.size > 0; + const hasArtifacts = content.releases.some((r) => r.artifact_id); + const selectedCount = content.releases.filter( + (r) => r.is_latest === true + ).length; + const fail = (message: string): DatabaseError => ({ + code: "CONFLICT", + message + }); + if (selectedCount > 1) return fail("Select only one latest release"); + if ( + !managed && + !hasArtifacts && + !selectedCount && + !content.releases.some((r) => r.sha256) + ) + return null; + const latest = latestRelease(content.releases); + if ( + !latest || + latest.tag.replace(/^v/, "") !== content.version || + latest.download_url !== content.download_url || + canonicalJson(latest.requires) !== canonicalJson(content.requires) + ) + return fail( + "Version, download and requirements must match the selected latest release" + ); + if ((managed || hasArtifacts) && !latest.artifact_id) + return fail( + "The latest release needs a directory-built package; upload its source ZIP first" + ); + const artifactIds = [ + ...new Set( + content.releases.flatMap((release) => + release.artifact_id ? [release.artifact_id] : [] + ) + ) + ]; + const rows = await db + .prepare( + `${ARTIFACT_SELECT} JOIN developers d ON d.profile_generation=s.profile_generation WHERE a.id IN (SELECT value FROM json_each(?)) AND s.extension_identifier=? AND d.id=?` + ) + .bind(JSON.stringify(artifactIds), id, developerId) + .all(); + const artifacts = new Map(rows.results.map((row) => [row.id, row])); + const versions = new Set(); + for (const release of content.releases) { + const version = release.tag.replace(/^v/, ""); + if ((managed || hasArtifacts) && versions.has(version)) + return fail("Duplicate release version"); + versions.add(version); + if (!release.artifact_id) { + if (release.sha256) + return fail("Checksums require a directory-built artifact"); + if (publishedVersions.has(version)) + return fail( + "A directory-built release cannot be replaced with an external download" + ); + continue; + } + const artifact = artifacts.get(release.artifact_id); + if ( + !artifact || + artifact.version !== version || + artifact.download_url !== release.download_url || + artifact.sha256 !== release.sha256 + ) + return fail("Release does not match its package artifact"); + const manifest = ManifestSchema.parse(JSON.parse(artifact.manifest)); + if (canonicalJson(manifest.requires) !== canonicalJson(release.requires)) + return fail("Release requirements do not match its package manifest"); + if ( + release === latest && + (content.download_url !== artifact.download_url || + canonicalJson(manifest) !== canonicalJson(contentManifest(id, content))) + ) + return fail( + "Overview metadata must match the latest package manifest; rebuild it before submitting" + ); + } + return null; +} + +export function packageIconUrl(content: ExtensionContent): string | null { + return latestRelease(content.releases)?.artifact_id + ? content.download_url.replace(/\/download$/, "/icon") + : null; +} diff --git a/src/services/extensions/v2/db/revisions.ts b/src/services/extensions/v2/db/revisions.ts index 83df813e..5799328d 100644 --- a/src/services/extensions/v2/db/revisions.ts +++ b/src/services/extensions/v2/db/revisions.ts @@ -1,3 +1,4 @@ +import { packageIconUrl, validatePackageContent } from "./packages"; import { manifestMetadata } from "../schemas/manifest"; import { MAX_CONTENT_BYTES } from "../resource-limits"; import { and, asc, desc, eq, gt, lt, or, sql, SQL } from "drizzle-orm"; @@ -151,6 +152,14 @@ export class ExtensionRevisionsDatabase { let result; try { + const packageError = await validatePackageContent( + this.db.$client, + input.extensionId, + undefined, + input.content, + input.callerId + ); + if (packageError) return { data: null, error: packageError }; result = await this.db.all<{ identifier: string }>(sql` INSERT INTO ${extensionRevisions} (id, extension_id, developer_id, submitted_by, status, content, ownership_epoch) @@ -196,6 +205,14 @@ export class ExtensionRevisionsDatabase { }): Promise> { const id = crypto.randomUUID(); try { + const packageError = await validatePackageContent( + this.db.$client, + input.extensionId, + undefined, + input.content, + input.callerId + ); + if (packageError) return { data: null, error: packageError }; const results = await this.db.$client.batch<{ identifier: string }>([ this.db.$client .prepare( @@ -335,6 +352,13 @@ export class ExtensionRevisionsDatabase { let result; try { + const packageError = await validatePackageContent( + this.db.$client, + input.extensionId, + input.developerId, + input.content + ); + if (packageError) return { data: null, error: packageError }; result = await this.db.all<{ identifier: string }>(sql` INSERT INTO ${extensionRevisions} (id, extension_id, developer_id, submitted_by, status, content, ownership_epoch) @@ -794,6 +818,20 @@ export class ExtensionRevisionsDatabase { } }; } + try { + const packageError = await validatePackageContent( + this.db.$client, + revision.extension_identifier, + revision.developer_id, + parsed.data + ); + if (packageError) return { ok: false, error: packageError }; + } catch (error) { + return { + ok: false, + error: databaseError("preparePackageApproval", error).error! + }; + } return { ok: true, revision, content: parsed.data }; } @@ -837,7 +875,7 @@ export class ExtensionRevisionsDatabase { JSON.stringify(content.releases), content.homepage_url ?? "", JSON.stringify(content.license), - content.icon_url ?? null, + packageIconUrl(content), content.readme, JSON.stringify(content.source), content.version, diff --git a/src/services/extensions/v2/db/schema.ts b/src/services/extensions/v2/db/schema.ts index ea15aaa2..53630359 100644 --- a/src/services/extensions/v2/db/schema.ts +++ b/src/services/extensions/v2/db/schema.ts @@ -150,16 +150,7 @@ export const developers = sqliteTable( url: text("url"), ownerUserId: text("owner_user_id").references(() => users.id), approvedAt: text("approved_at"), - // Placeholder default from migration 0002 (SQLite rejects non-constant - // ALTER TABLE ADD COLUMN defaults). Every write sets this explicitly (see - // db/developer-profiles.ts) - the literal default is never actually read, - // but it is part of the real column definition, so it is kept here for - // baseline-diff fidelity against the existing database. - // - // Replacing it needs a table rebuild, which this table cannot have: three - // tables reference it and a parent cannot be dropped with foreign keys - // enforced (see migration 0021's header). Rebuilding all three children - // too is a lot of risk for a default nothing reads. + // Kept in sync with the original constant database default. Writers set both timestamps. createdAt: text("created_at").notNull().default("1970-01-01T00:00:00.000Z"), updatedAt: text("updated_at").notNull().default("1970-01-01T00:00:00.000Z"), avatarUrl: text("avatar_url"), @@ -516,3 +507,134 @@ export const m2mPublisherClients = sqliteTable( ) ] ); + +// Object reservations survive interrupted uploads so unreferenced objects can +// be reclaimed. Published artifacts retain their original objects indefinitely. +export const packageObjects = sqliteTable( + "package_objects", + { + deleting: integer("deleting").notNull().default(0), + id: text("id").primaryKey(), + createdBy: text("created_by").notNull(), + bytes: integer("bytes").notNull(), + createdAt: text("created_at") + .notNull() + .default(sql`CURRENT_TIMESTAMP`) + }, + (table) => [index("idx_package_objects_creator").on(table.createdBy)] +); + +// No extension FK: a developer can build a package before creating its listing. +export const packageSources = sqliteTable( + "package_sources", + { + id: text("id").primaryKey(), + profileGeneration: text("profile_generation").notNull(), + extensionIdentifier: text("extension_identifier").notNull(), + version: text("version").notNull(), + sha256: text("sha256").notNull(), + objectId: text("object_id") + .notNull() + .references(() => packageObjects.id), + createdAt: text("created_at") + .notNull() + .default(sql`CURRENT_TIMESTAMP`) + }, + (table) => [ + uniqueIndex("idx_package_source_version").on( + table.profileGeneration, + table.extensionIdentifier, + table.version + ), + index("idx_package_sources_extension").on(table.extensionIdentifier), + uniqueIndex("idx_package_sources_object").on(table.objectId) + ] +); + +export const packageArtifacts = sqliteTable( + "package_artifacts", + { + id: text("id").primaryKey(), + sourceId: text("source_id") + .notNull() + .references(() => packageSources.id), + manifest: text("manifest").notNull(), + manifestHash: text("manifest_hash").notNull(), + sha256: text("sha256").notNull(), + objectId: text("object_id") + .notNull() + .references(() => packageObjects.id), + downloadUrl: text("download_url").notNull(), + publishedAt: text("published_at"), + createdAt: text("created_at") + .notNull() + .default(sql`CURRENT_TIMESTAMP`) + }, + (table) => [ + uniqueIndex("idx_package_artifact_manifest").on( + table.sourceId, + table.manifestHash + ), + uniqueIndex("idx_package_artifacts_object").on(table.objectId) + ] +); + +// Prepared display images are shared by packages using the same immutable source. +export const packageIcons = sqliteTable("package_icons", { + sourceId: text("source_id") + .primaryKey() + .references(() => packageSources.id), + avifObjectId: text("avif_object_id").references(() => packageObjects.id), + avifSha256: text("avif_sha256"), + pngObjectId: text("png_object_id").references(() => packageObjects.id), + pngSha256: text("png_sha256"), + svgObjectId: text("svg_object_id").references(() => packageObjects.id), + svgSha256: text("svg_sha256") +}); + +export const packageJobs = sqliteTable( + "package_jobs", + { + id: text("id").primaryKey(), + fingerprint: text("fingerprint").notNull(), + ownerUserId: text("owner_user_id").notNull(), + owner: text("owner").notNull(), + manifest: text("manifest").notNull(), + sourceId: text("source_id"), + uploadObjectId: text("upload_object_id").references( + () => packageObjects.id + ), + uploadSha256: text("upload_sha256"), + status: text("status", { + enum: ["queued", "processing", "ready", "failed"] + }) + .notNull() + .default("queued"), + attempts: integer("attempts").notNull().default(0), + leaseId: text("lease_id"), + leaseUntil: text("lease_until"), + artifactId: text("artifact_id").references(() => packageArtifacts.id), + errorCode: text("error_code"), + errorMessage: text("error_message"), + errorStatus: integer("error_status"), + enqueuedAt: text("enqueued_at"), + createdAt: text("created_at") + .notNull() + .default(sql`CURRENT_TIMESTAMP`), + updatedAt: text("updated_at") + .notNull() + .default(sql`CURRENT_TIMESTAMP`) + }, + (table) => [ + uniqueIndex("idx_package_jobs_fingerprint") + .on(table.fingerprint) + .where(sql`${table.status} != 'failed'`), + index("idx_package_jobs_owner").on(table.ownerUserId), + index("idx_package_jobs_upload_object").on(table.uploadObjectId), + index("idx_package_jobs_dispatch").on(table.status, table.enqueuedAt), + check( + "package_jobs_status_check", + sql`${table.status} IN ('queued','processing','ready','failed')` + ) + ] +); diff --git a/src/services/extensions/v2/index.ts b/src/services/extensions/v2/index.ts index 948fcc8f..ce7a76b0 100644 --- a/src/services/extensions/v2/index.ts +++ b/src/services/extensions/v2/index.ts @@ -1,3 +1,4 @@ +import { registerPackageRoutes } from "./routes/packages"; import { boundContentRequest, paceContentIp, @@ -90,6 +91,7 @@ registerOwnershipRoutes(extensionsV2); registerModerationRoutes(extensionsV2); registerM2mRoutes(extensionsV2); registerRevisionRoutes(extensionsV2); +registerPackageRoutes(extensionsV2); registerDeveloperProfileRoutes(extensionsV2); // The document is deterministic once every route is registered, so build diff --git a/src/services/extensions/v2/package-builder.ts b/src/services/extensions/v2/package-builder.ts new file mode 100644 index 00000000..59aa497d --- /dev/null +++ b/src/services/extensions/v2/package-builder.ts @@ -0,0 +1,209 @@ +import { + Uint8ArrayReader, + Uint8ArrayWriter, + ZipReader, + ZipWriter +} from "@zip.js/zip.js"; +import { ManifestSchema } from "./schemas/manifest"; + +// Bounds for uploaded ZIPs and their decompressed contents. +export const PACKAGE_LIMITS = { + sourceBytes: 5 * 1024 * 1024, + expandedBytes: 16 * 1024 * 1024, + entries: 500, + pathLength: 1024, + pathDepth: 32 +} as const; + +const zipOptions = { + useWebWorkers: false, + useCompressionStream: true, + strictness: "strict" as const, + checkCrc32: true, + checkOverlappingEntry: true +}; + +export async function sha256(bytes: Uint8Array) { + const digest = await crypto.subtle.digest("SHA-256", bytes); + return Array.from(new Uint8Array(digest), (byte) => + byte.toString(16).padStart(2, "0") + ).join(""); +} + +function validatePath(filename: string, directory: boolean) { + const path = directory ? filename.replace(/\/$/, "") : filename; + if (path.length > PACKAGE_LIMITS.pathLength) + throw new Error("Package path exceeds length limit"); + const parts = path.split("/"); + if (parts.length > PACKAGE_LIMITS.pathDepth) + throw new Error("Package path exceeds depth limit"); + if ( + !path || + /[\\:<>"|?*]/.test(path) || + [...path].some( + (char) => char.charCodeAt(0) < 32 || char.charCodeAt(0) === 127 + ) || + parts.some((part) => !part || part === "." || part === "..") || + parts.some( + (part) => + /[. ]$/.test(part) || + /^(con|prn|aux|nul|com[1-9]|lpt[1-9])(?:\.|$)/i.test(part) + ) + ) { + throw new Error(`Unsafe package path: ${filename}`); + } + return path; +} + +/** Validate source files and build a bounded archive with directory metadata. */ +export async function buildExtensionPackage( + source: Uint8Array, + manifestInput: unknown +) { + const manifest = ManifestSchema.parse(manifestInput); + if (source.byteLength > PACKAGE_LIMITS.sourceBytes) { + throw new Error("Source ZIP exceeds size limit"); + } + const reader = new ZipReader(new Uint8ArrayReader(source), zipOptions); + const files: { + path: string; + bytes: Uint8Array; + compression?: { + passThrough: true; + compressionMethod: number; + uncompressedSize: number; + crc32?: number; + }; + icon?: Uint8Array; + }[] = []; + const paths = new Map(); + let count = 0; + let expandedBytes = 0; + try { + for await (const entry of reader.getEntriesGenerator()) { + if (++count > PACKAGE_LIMITS.entries) { + throw new Error("Source ZIP exceeds entry limit"); + } + const path = validatePath(entry.filename, entry.directory); + // Avoid case-folding collisions across supported installation platforms. + const key = path.normalize("NFC").toLowerCase(); + if (key.startsWith("manifest.json/")) { + throw new Error("Manifest path is a directory"); + } + if (paths.has(key)) throw new Error(`Duplicate package path: ${path}`); + paths.set(key, entry.directory); + const fileType = (entry.externalFileAttributes >>> 16) & 0o170000; + if ( + entry.encrypted || + (fileType !== 0 && + fileType !== (entry.directory ? 0o040000 : 0o100000)) || + ![0, 8].includes(entry.compressionMethod) + ) { + throw new Error(`Unsupported package entry: ${path}`); + } + if ( + entry.uncompressedSize > + PACKAGE_LIMITS.expandedBytes - expandedBytes + ) { + throw new Error("Source ZIP exceeds expanded size limit"); + } + if (entry.directory) continue; + // Bound the actual decompressed bytes, not just the untrusted ZIP header. + const isIcon = /^icon\.(svg|png|webp|avif)$/.test(path); + const chunks: Uint8Array[] = []; + let length = 0; + await entry.getData( + new WritableStream({ + write(chunk) { + expandedBytes += chunk.byteLength; + length += chunk.byteLength; + if (expandedBytes > PACKAGE_LIMITS.expandedBytes) { + throw new Error("Source ZIP exceeds expanded size limit"); + } + if (isIcon) chunks.push(chunk.slice()); + } + }) + ); + if (length !== entry.uncompressedSize) { + throw new Error("Source ZIP entry has inconsistent size"); + } + const icon = isIcon ? new Uint8Array(length) : undefined; + let offset = 0; + for (const chunk of chunks) { + icon!.set(chunk, offset); + offset += chunk.byteLength; + } + // Validate the existing manifest's ZIP entry too, then replace its bytes. + if (key === "manifest.json") { + if (path !== "manifest.json") { + throw new Error("Manifest must be named manifest.json"); + } + } else { + // CRC and expanded-size validation above still decompresses every file. + // Copy its original compressed payload instead of compressing it again. + const bytes = await entry.getData(new Uint8ArrayWriter(), { + passThrough: true + }); + files.push({ + path, + bytes, + icon, + compression: { + passThrough: true, + compressionMethod: entry.compressionMethod, + uncompressedSize: entry.uncompressedSize, + crc32: entry.signature + } + }); + } + } + } finally { + await reader.close(); + } + for (const [path] of paths) { + const parts = path.split("/"); + parts.pop(); + while (parts.length) { + if (paths.get(parts.join("/")) === false) { + throw new Error(`File conflicts with package directory: ${path}`); + } + parts.pop(); + } + } + if (paths.get("manifest.json") === true) { + throw new Error("Manifest path is a directory"); + } + if (!files.length) throw new Error("Package contains no source files"); + const icons = files.filter((file) => + /^icon\.(svg|png|webp|avif)$/.test(file.path) + ); + if (icons.length > 1) throw new Error("Include only one package icon"); + + const manifestBytes = new Uint8Array( + new TextEncoder().encode(JSON.stringify(manifest, null, 2) + "\n") + ); + files.push({ path: "manifest.json", bytes: manifestBytes }); + files.sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0)); + const writer = new ZipWriter(new Uint8ArrayWriter(), { + ...zipOptions, + level: 6, + lastModDate: new Date("1980-01-01T00:00:00Z"), + extendedTimestamp: false + }); + for (const file of files) { + await writer.add( + file.path, + new Uint8ArrayReader(file.bytes), + file.compression + ); + } + const bytes = await writer.close(); + return { + bytes, + manifest, + sourceSha256: await sha256(source), + packageSha256: await sha256(bytes), + icon: icons[0]?.icon ?? null, + iconName: icons[0]?.path + }; +} diff --git a/src/services/extensions/v2/packages/build.ts b/src/services/extensions/v2/packages/build.ts new file mode 100644 index 00000000..371aef56 --- /dev/null +++ b/src/services/extensions/v2/packages/build.ts @@ -0,0 +1,215 @@ +import { + buildExtensionPackage, + PACKAGE_LIMITS, + sha256 +} from "../package-builder"; +import type { Manifest } from "../schemas/manifest"; +import { canonicalJson } from "./manifest"; +import type { PackageRepository, PackageOwner, IconObject } from "./repository"; +import type { PackageImageProcessor, IconVariant } from "./images"; +import { PackageError } from "./errors"; +import { boundedBytes, type PackageStore } from "./store"; + +export async function reserveObject( + repository: PackageRepository, + store: PackageStore, + owner: PackageOwner, + identifier: string, + bytes: Uint8Array, + checksum: string, + id: string = crypto.randomUUID() +) { + await repository.reserveObject(owner, identifier, id, bytes.length); + // Failed writes retain their reservation for later cleanup. + await store.put(id, bytes, checksum); + return id; +} +export async function cleanupPackageObjects( + repository: PackageRepository, + store: PackageStore +) { + const ids = await repository.claimCleanup(); + for (const id of ids) { + await store.delete(id); + await repository.finishCleanup(id); + } + return ids.length; +} + +export async function buildStoredPackage(input: { + repository: PackageRepository; + store: PackageStore; + owner: PackageOwner; + manifest: Manifest; + upload?: { bytes: Uint8Array; objectId: string }; + sourceId?: string; + baseUrl: string; + images: PackageImageProcessor; +}) { + const { repository, store, owner, manifest } = input; + let source = await repository.findSource( + owner.profile_generation, + manifest.id, + manifest.version + ); + if (input.sourceId && source?.id !== input.sourceId) + throw new PackageError( + "NOT_FOUND", + "Source not found for this extension version", + 404 + ); + let sourceBytes = input.upload?.bytes; + if (sourceBytes && source && (await sha256(sourceBytes)) !== source.sha256) + throw new PackageError( + "SOURCE_CHANGED", + "This version already has a different source ZIP; use a new extension version", + 409 + ); + const manifestJson = canonicalJson(manifest); + const manifestHash = await sha256( + new Uint8Array(new TextEncoder().encode(manifestJson)) + ); + if (source) { + const existing = await repository.findArtifact(source.id, manifestHash); + if (existing) return existing; + } + if (!sourceBytes) { + if (!source) + throw new PackageError("NOT_FOUND", "Source ZIP not found", 404); + sourceBytes = await boundedBytes( + await store.get(source.object_id), + PACKAGE_LIMITS.sourceBytes + ); + if ((await sha256(sourceBytes)) !== source.sha256) + throw new PackageError( + "SOURCE_UNAVAILABLE", + "Stored source failed checksum verification", + 503 + ); + } + let built; + try { + built = await buildExtensionPackage(sourceBytes, manifest); + } catch { + throw new PackageError( + "INVALID_PACKAGE", + "Source ZIP failed package validation", + 422 + ); + } + const prepared = source && (await repository.hasIcon(source.id)); + const variants = + prepared || !built.icon + ? [] + : await input.images.prepare(built.icon, built.iconName === "icon.svg"); + if (!source) { + // Without an upload, the stored-source lookup above must have succeeded. + const objectId = input.upload!.objectId; + source = await repository.linkSource(owner, manifest, { + id: crypto.randomUUID(), + sha256: built.sourceSha256, + object_id: objectId + }); + if (!source) + throw new PackageError( + "FORBIDDEN", + "Ownership changed while building", + 403 + ); + if (source.sha256 !== built.sourceSha256) + throw new PackageError( + "SOURCE_CHANGED", + "Another upload reserved this version with different source files", + 409 + ); + } + + const extensionId = await repository.ensureDraft(owner, manifest.id); + await storePackageIcon({ + repository, + extensionId, + store, + owner, + manifest, + sourceId: source.id, + variants + }); + const objectId = await reserveObject( + repository, + store, + owner, + manifest.id, + built.bytes, + built.packageSha256 + ); + const artifactId = crypto.randomUUID(); + const downloadUrl = `${input.baseUrl}/packages/${artifactId}/download`; + const artifact = await repository.linkArtifact(owner, manifest, { + id: artifactId, + sourceId: source.id, + manifestHash, + sha256: built.packageSha256, + objectId, + downloadUrl + }); + if (!artifact) + throw new PackageError( + "FORBIDDEN", + "Ownership changed while building", + 403 + ); + if (artifact.object_id !== objectId) { + await store.delete(objectId); + await repository.removeObject(objectId); + } + return artifact; +} + +async function storePackageIcon(input: { + extensionId: string; + repository: PackageRepository; + store: PackageStore; + owner: PackageOwner; + manifest: Manifest; + sourceId: string; + variants: IconVariant[]; +}) { + const { repository, store, owner, manifest, sourceId, variants } = input; + if (await repository.hasIcon(sourceId)) return; + const assetId = crypto.randomUUID(); + const objects: IconObject[] = []; + for (const { format, bytes } of variants) { + const checksum = await sha256(bytes); + objects.push({ + format, + id: await reserveObject( + repository, + store, + owner, + manifest.id, + bytes, + checksum, + `extensions/${input.extensionId}/icons/${assetId}.${format}` + ), + checksum + }); + } + const linked = await repository.linkIcon( + owner, + manifest.id, + sourceId, + objects + ); + if (!linked) { + for (const object of objects) { + await store.delete(object.id); + await repository.removeObject(object.id); + } + if (!(await repository.hasIcon(sourceId))) + throw new PackageError( + "FORBIDDEN", + "Ownership changed while processing icon", + 403 + ); + } +} diff --git a/src/services/extensions/v2/packages/errors.ts b/src/services/extensions/v2/packages/errors.ts new file mode 100644 index 00000000..333a574d --- /dev/null +++ b/src/services/extensions/v2/packages/errors.ts @@ -0,0 +1,9 @@ +export class PackageError extends Error { + constructor( + public code: string, + message: string, + public status: 403 | 404 | 409 | 413 | 422 | 429 | 503 + ) { + super(message); + } +} diff --git a/src/services/extensions/v2/packages/images.ts b/src/services/extensions/v2/packages/images.ts new file mode 100644 index 00000000..efc41174 --- /dev/null +++ b/src/services/extensions/v2/packages/images.ts @@ -0,0 +1,10 @@ +export interface IconVariant { + format: "avif" | "png" | "svg"; + bytes: Uint8Array; +} +export interface PackageImageProcessor { + prepare( + icon: Uint8Array, + isSvg: boolean + ): Promise; +} diff --git a/src/services/extensions/v2/packages/jobs.ts b/src/services/extensions/v2/packages/jobs.ts new file mode 100644 index 00000000..0af883d0 --- /dev/null +++ b/src/services/extensions/v2/packages/jobs.ts @@ -0,0 +1,87 @@ +import { logError } from "../../../../lib/logger"; +import type { IMessageQueue } from "../../../../lib/interfaces"; +import type { Manifest } from "../schemas/manifest"; +import type { PackageOwner } from "./repository"; +import { PackageError } from "./errors"; + +export interface PackageJob { + id: string; + fingerprint: string; + owner: PackageOwner; + manifest: Manifest; + sourceId: string | null; + uploadObjectId: string | null; + uploadSha256: string | null; + status: "queued" | "processing" | "ready" | "failed"; + attempts: number; + artifactId: string | null; + error: { code: string; message: string; status: number } | null; + createdAt: string; + updatedAt: string; +} + +export interface PackageJobRepository { + get(id: string): Promise; + claim(id: string, lease: string): Promise; + complete(id: string, lease: string, artifactId: string): Promise; + fail( + id: string, + lease: string, + error: NonNullable, + retry: boolean + ): Promise; + pending(): Promise; + markEnqueued(id: string): Promise; +} + +export async function dispatchPackageJob( + jobs: PackageJobRepository, + queue: IMessageQueue, + id: string +) { + await queue.send({ jobId: id }); + await jobs.markEnqueued(id); +} + +// Delivery-specific acknowledgement/retry APIs stay in the platform adapter. +export async function processPackageJob( + jobs: PackageJobRepository, + id: string, + prepare: (job: PackageJob) => Promise, + lease = crypto.randomUUID() +): Promise<"ack" | "retry" | "busy"> { + const existing = await jobs.get(id); + if (!existing || existing.status === "ready" || existing.status === "failed") + return "ack"; + const job = await jobs.claim(id, lease); + if (!job) return "busy"; + try { + if (job.attempts > 3) + throw new PackageError( + "PACKAGE_UNAVAILABLE", + "Package processing was interrupted repeatedly. Please try again.", + 503 + ); + const artifactId = await prepare(job); + await jobs.complete(id, lease, artifactId); + return "ack"; + } catch (cause) { + if (!(cause instanceof PackageError)) + logError("extensions/packages", "Package preparation failed", { + jobId: id, + error: cause instanceof Error ? cause.message : String(cause) + }); + const error = + cause instanceof PackageError + ? { code: cause.code, message: cause.message, status: cause.status } + : { + code: "PACKAGE_UNAVAILABLE", + message: + "Package processing is temporarily unavailable. Please try again.", + status: 503 + }; + const retry = error.status === 503 && job.attempts < 3; + await jobs.fail(id, lease, error, retry); + return retry ? "retry" : "ack"; + } +} diff --git a/src/services/extensions/v2/packages/manifest.ts b/src/services/extensions/v2/packages/manifest.ts new file mode 100644 index 00000000..b0d36b92 --- /dev/null +++ b/src/services/extensions/v2/packages/manifest.ts @@ -0,0 +1,29 @@ +import type { ExtensionContent } from "../schemas/extensions"; +import { ManifestSchema } from "../schemas/manifest"; + +// Sort nested maps too: object key ordering does not change a manifest. +export function canonicalJson(value: unknown): string { + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`; + if (value !== null && typeof value === "object") + return `{${Object.entries(value) + .filter(([, v]) => v !== undefined) + .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) + .map(([k, v]) => `${JSON.stringify(k)}:${canonicalJson(v)}`) + .join(",")}}`; + return JSON.stringify(value); +} + +export function contentManifest(id: string, content: ExtensionContent) { + return ManifestSchema.parse({ + id, + manifest_version: content.manifest_version, + type: content.type, + name: content.name, + version: content.version, + description: content.description, + author: content.author, + license: content.license, + homepage_url: content.homepage_url, + requires: content.requires + }); +} diff --git a/src/services/extensions/v2/packages/prepare.ts b/src/services/extensions/v2/packages/prepare.ts new file mode 100644 index 00000000..35061266 --- /dev/null +++ b/src/services/extensions/v2/packages/prepare.ts @@ -0,0 +1,72 @@ +import type { PackageRepository } from "./repository"; +import type { PackageImageProcessor } from "./images"; +import type { PackageStore } from "./store"; +import type { PackageJob } from "./jobs"; +import { PackageError } from "./errors"; +import { boundedBytes } from "./store"; +import { buildStoredPackage } from "./build"; +import { PACKAGE_LIMITS } from "../package-builder"; + +export interface PackagePreparation { + repository: PackageRepository; + store: PackageStore; + images: PackageImageProcessor; + baseUrl: string; +} + +export async function prepareQueuedPackage( + services: PackagePreparation, + job: PackageJob +): Promise { + const { repository, store } = services; + const previous = job.owner; + const current = await repository.getOwner( + previous.owner_user_id, + job.manifest.id, + previous.publisher_client_id + ? { + clientId: previous.publisher_client_id, + developerId: previous.id, + ownershipEpoch: previous.ownership_epoch + } + : null + ); + if ( + current.id !== previous.id || + current.profile_generation !== previous.profile_generation || + current.ownership_epoch !== previous.ownership_epoch + ) + throw new PackageError( + "FORBIDDEN", + "Package ownership changed while queued", + 403 + ); + let upload: { bytes: Uint8Array; objectId: string } | undefined; + if (job.uploadObjectId) { + const body = await store.get( + job.uploadObjectId, + job.uploadSha256 ?? undefined + ); + if (!body) + throw new PackageError( + "SOURCE_UNAVAILABLE", + "Queued source ZIP is unavailable", + 503 + ); + upload = { + bytes: await boundedBytes(body, PACKAGE_LIMITS.sourceBytes), + objectId: job.uploadObjectId + }; + } + const artifact = await buildStoredPackage({ + repository, + store, + owner: current, + manifest: job.manifest, + upload, + sourceId: job.sourceId ?? undefined, + baseUrl: services.baseUrl.replace(/\/$/, ""), + images: services.images + }); + return artifact.id; +} diff --git a/src/services/extensions/v2/packages/repository.ts b/src/services/extensions/v2/packages/repository.ts new file mode 100644 index 00000000..a43e097c --- /dev/null +++ b/src/services/extensions/v2/packages/repository.ts @@ -0,0 +1,95 @@ +import type { Manifest } from "../schemas/manifest"; + +export interface PackageOwner { + id: string; + profile_generation: string; + ownership_epoch: number; + owner_user_id: string; + publisher_client_id?: string; +} +export interface SourceRow { + id: string; + sha256: string; + object_id: string; +} +export interface ArtifactRow { + id: string; + source_id: string; + manifest: string; + sha256: string; + object_id: string; + download_url: string; + published_at: string | null; + created_at: string; + extension_identifier: string; + profile_generation: string; + version: string; + icon_source_id: string | null; + avif_object_id: string | null; + avif_sha256: string | null; + png_object_id: string | null; + png_sha256: string | null; + svg_object_id: string | null; + svg_sha256: string | null; +} + +export interface PackagePublisher { + clientId: string; + developerId: string; + ownershipEpoch: number; +} +export interface IconObject { + format: "svg" | "avif" | "png"; + id: string; + checksum: string; +} +export interface PackageRepository { + getOwner( + userId: string, + identifier: string, + publisher?: PackagePublisher | null + ): Promise; + findSource( + generation: string, + identifier: string, + version: string + ): Promise; + findArtifact( + sourceId: string, + manifestHash: string + ): Promise; + hasIcon(sourceId: string): Promise; + reserveObject( + owner: PackageOwner, + identifier: string, + id: string, + bytes: number + ): Promise; + removeObject(id: string): Promise; + claimCleanup(): Promise; + finishCleanup(id: string): Promise; + linkSource( + owner: PackageOwner, + manifest: Manifest, + source: SourceRow + ): Promise; + ensureDraft(owner: PackageOwner, identifier: string): Promise; + linkArtifact( + owner: PackageOwner, + manifest: Manifest, + artifact: { + id: string; + sourceId: string; + manifestHash: string; + sha256: string; + objectId: string; + downloadUrl: string; + } + ): Promise; + linkIcon( + owner: PackageOwner, + identifier: string, + sourceId: string, + objects: IconObject[] + ): Promise; +} diff --git a/src/services/extensions/v2/packages/store.ts b/src/services/extensions/v2/packages/store.ts new file mode 100644 index 00000000..5f5901fd --- /dev/null +++ b/src/services/extensions/v2/packages/store.ts @@ -0,0 +1,49 @@ +import { PackageError } from "./errors"; + +export interface PackageStore { + put( + key: string, + bytes: Uint8Array, + checksum: string + ): Promise; + get( + key: string, + checksum?: string + ): Promise | null>; + delete(key: string): Promise; +} + +export async function boundedBytes( + body: ReadableStream | null, + limit: number +) { + if (!body) return new Uint8Array(0); + const reader = body.getReader(); + const chunks: Uint8Array[] = []; + let size = 0; + try { + for (;;) { + const { value, done } = await reader.read(); + if (done) break; + size += value.byteLength; + if (size > limit) { + await reader.cancel(); + throw new PackageError( + "BODY_TOO_LARGE", + "Upload exceeds size limit", + 413 + ); + } + chunks.push(value); + } + } finally { + reader.releaseLock(); + } + const bytes = new Uint8Array(size); + let offset = 0; + for (const chunk of chunks) { + bytes.set(chunk, offset); + offset += chunk.byteLength; + } + return bytes; +} diff --git a/src/services/extensions/v2/routes/moderation.ts b/src/services/extensions/v2/routes/moderation.ts index e6a2dff7..ebdde3a3 100644 --- a/src/services/extensions/v2/routes/moderation.ts +++ b/src/services/extensions/v2/routes/moderation.ts @@ -1,3 +1,4 @@ +import { schedulePublishedIconUpdate } from "../../../../lib/adapters/cloudflare/package-assets"; import { paceContentAccount, requireApproveAccess, @@ -118,6 +119,7 @@ export function registerModerationRoutes(app: ExtensionsV2App): void { const status = statusFromWriteErrorCode(error?.code); return c.json(errorBody(error, "Unable to approve revision"), status); } + schedulePublishedIconUpdate(c, id); const notified = await notifyAuthor(c, extDb, query, { kind: "revision-approved", extensionId: id, @@ -416,6 +418,7 @@ export function registerModerationRoutes(app: ExtensionsV2App): void { const status = statusFromContentWriteError(error?.code); return c.json(errorBody(error, "Unable to correct extension"), status); } + schedulePublishedIconUpdate(c, id); revalidateExtensions(c); return c.json( { diff --git a/src/services/extensions/v2/routes/packages.ts b/src/services/extensions/v2/routes/packages.ts new file mode 100644 index 00000000..1cdacdd9 --- /dev/null +++ b/src/services/extensions/v2/routes/packages.ts @@ -0,0 +1,501 @@ +import { PackageJobsDatabase } from "../db/package-jobs"; +import { dispatchPackageJob, type PackageJob } from "../packages/jobs"; +import { sha256 } from "../package-builder"; +import { createRoute, z } from "@hono/zod-openapi"; +import type { Context, MiddlewareHandler } from "hono"; +import { getAuth } from "../../../../lib/auth"; +import { getPlatform } from "../../../../lib/middleware"; +import { canonicalJson } from "../packages/manifest"; +import type { ArtifactRow } from "../packages/repository"; +import { PackageBuildDatabase } from "../db/package-builds"; +import { + requireContentWrite, + getPublisher, + requireModerator, + optionalAuth, + getOptionalAuth, + paceContentAccount, + paceContentIp +} from "../middleware"; +import { reserveObject, cleanupPackageObjects } from "../packages/build"; +import { PackageError } from "../packages/errors"; +import { boundedBytes } from "../packages/store"; +import { packageStore } from "../../../../lib/adapters/cloudflare/package-store"; +import { ManifestSchema } from "../schemas/manifest"; +import { + PackageArtifactSchema, + PackageJobSchema, + PackageJobParamSchema, + PackageBuildSchema, + PackageParamSchema +} from "../schemas/packages"; +import { errorResponse, authErrorResponses } from "../schemas/common"; +import { ExtensionsV2App } from "./app"; + +function failure(c: Context, error: unknown) { + if (error instanceof PackageError) + return c.json( + { error: { code: error.code, message: error.message } }, + error.status + ); + console.error("[packages] Package operation failed"); + return c.json( + { + error: { + code: "PACKAGE_UNAVAILABLE", + message: "Package storage is temporarily unavailable; retry the request" + } + }, + 503 + ); +} +function artifactResponse(row: ArtifactRow) { + return { + result: { + id: row.id, + source_id: row.source_id, + manifest: ManifestSchema.parse(JSON.parse(row.manifest)), + sha256: row.sha256, + download_url: row.download_url, + published_at: row.published_at, + created_at: row.created_at + } + }; +} +async function jobResponse(db: D1Database, job: PackageJob) { + const artifact = job.artifactId + ? await new PackageBuildDatabase(db).getArtifact(job.artifactId) + : null; + return { + result: { + id: job.id, + extension_identifier: job.manifest.id, + status: job.status, + artifact: artifact ? artifactResponse(artifact).result : null, + error: job.status === "failed" ? job.error : null, + created_at: job.createdAt, + updated_at: job.updatedAt + } + }; +} + +const packageErrors = { + ...authErrorResponses(), + 404: errorResponse("Package not found"), + 409: errorResponse( + "Source version conflict, ownership changed, or storage quota exhausted" + ), + 413: errorResponse("Upload exceeds size limit"), + 422: errorResponse("Invalid manifest or source ZIP"), + 429: errorResponse("Write allowance exhausted"), + 503: errorResponse("Package storage is not configured or is unavailable") +}; + +export function registerPackageRoutes(app: ExtensionsV2App) { + const availability: MiddlewareHandler<{ + Bindings: CloudflareBindings; + }> = async (c, next) => { + c.header("Cache-Control", "private, no-store"); + c.header("Vary", "Authorization"); + try { + if (!packageStore(c)) + throw new PackageError( + "PACKAGES_DISABLED", + "Package storage is not configured", + 503 + ); + } catch (error) { + return failure(c, error); + } + await next(); + }; + app.use("/packages", availability); + app.use("/packages/*", availability); + app.use("/package-jobs/*", availability); + app.on("POST", "/packages", paceContentIp()); + const boundUpload: MiddlewareHandler = async (c, next) => { + try { + const body = await boundedBytes(c.req.raw.body, 6 * 1024 * 1024); + c.req.raw = new Request(c.req.raw, { body }); + } catch (error) { + return failure(c, error); + } + await next(); + }; + + app.openapi( + createRoute({ + method: "post", + path: "/packages", + tags: ["Packages"], + summary: + "Queue package preparation from a distribution ZIP or stored source", + security: [{ Bearer: [] }], + middleware: [ + requireContentWrite(), + paceContentAccount(), + boundUpload + ] as const, + request: { + body: { + required: true, + content: { "multipart/form-data": { schema: PackageBuildSchema } } + } + }, + responses: { + 202: { + description: + "Private processing job; identical requests reuse queued or completed work", + content: { + "application/json": { + schema: z.object({ result: PackageJobSchema }) + } + } + }, + ...packageErrors + } + }), + async (c) => { + try { + const store = packageStore(c)!; + const form = c.req.valid("form"); + let json: unknown; + try { + json = JSON.parse(form.manifest); + } catch { + throw new PackageError( + "INVALID_MANIFEST", + "Manifest must be valid JSON", + 422 + ); + } + const parsed = ManifestSchema.safeParse(json); + if (!parsed.success) + throw new PackageError( + "INVALID_MANIFEST", + "Manifest failed validation", + 422 + ); + const manifest = parsed.data; + if (form.source && form.source.size > 5 * 1024 * 1024) + throw new PackageError( + "BODY_TOO_LARGE", + "Source ZIP exceeds 5 MiB", + 413 + ); + const db = c.env.DB_EXTENSIONS; + const repository = new PackageBuildDatabase(db); + const publisher = getPublisher(c); + let ownerUserId = getAuth(c).userId; + // Moderator corrections can rebuild published source, but cannot upload new files for another developer. + if (form.source_id && !publisher) { + const correction = await repository.correctionOwner( + ownerUserId, + form.source_id, + manifest + ); + if (correction) ownerUserId = correction.owner_user_id; + } + const owner = await repository.getOwner( + ownerUserId, + manifest.id, + publisher + ); + const baseUrl = getPlatform(c).getEnv("EXTENSION_PACKAGES_BASE_URL"); + if (!baseUrl) + throw new PackageError( + "PACKAGES_DISABLED", + "Package download URL is not configured", + 503 + ); + const jobs = new PackageJobsDatabase(db); + const source = form.source + ? new Uint8Array(await form.source.arrayBuffer()) + : undefined; + const uploadSha256 = source ? await sha256(source) : null; + const fingerprint = await sha256( + new Uint8Array( + new TextEncoder().encode( + canonicalJson({ + owner, + manifest, + source: uploadSha256 ?? form.source_id + }) + ) + ) + ); + let job = await jobs.find(fingerprint); + if (!job) { + const uploadObjectId = source + ? await reserveObject( + repository, + store, + owner, + manifest.id, + source, + uploadSha256! + ) + : null; + job = await jobs.create({ + fingerprint, + owner, + manifest, + sourceId: form.source_id ?? null, + uploadObjectId, + uploadSha256 + }); + } + if (job.status === "queued") + await dispatchPackageJob( + jobs, + getPlatform(c).getQueue("EXTENSION_PACKAGE_QUEUE"), + job.id + ); + c.header( + "Location", + `${baseUrl.replace(/\/$/, "")}/package-jobs/${job.id}` + ); + c.header("Retry-After", "1"); + return c.json(await jobResponse(db, job), 202); + } catch (error) { + return failure(c, error); + } + } + ); + + app.openapi( + createRoute({ + method: "get", + path: "/package-jobs/{jobId}", + tags: ["Packages"], + summary: "Read an owning developer's private package-processing job", + security: [{ Bearer: [] }], + middleware: [requireContentWrite()] as const, + request: { params: PackageJobParamSchema }, + responses: { + 200: { + description: "Package processing state and completed artifact", + content: { + "application/json": { + schema: z.object({ result: PackageJobSchema }) + } + } + }, + ...packageErrors + } + }), + async (c) => { + try { + const db = c.env.DB_EXTENSIONS; + const repository = new PackageBuildDatabase(db); + const job = await new PackageJobsDatabase(db).get( + c.req.valid("param").jobId + ); + if (!job) + throw new PackageError("NOT_FOUND", "Package job not found", 404); + const userId = getAuth(c).userId; + const moderator = await repository.isModerator(userId); + if (!moderator) { + const publisher = getPublisher(c); + const owner = await repository.getOwner( + userId, + job.manifest.id, + publisher + ); + if ( + owner.id !== job.owner.id || + owner.profile_generation !== job.owner.profile_generation || + owner.ownership_epoch !== job.owner.ownership_epoch || + (publisher && publisher.clientId !== job.owner.publisher_client_id) + ) + throw new PackageError("NOT_FOUND", "Package job not found", 404); + } + return c.json(await jobResponse(db, job), 200); + } catch (error) { + return failure(c, error); + } + } + ); + + // Registered before the parameter read. This only reclaims failed uploads; + // source/package rows and every published artifact remain immutable. + app.openapi( + createRoute({ + method: "post", + path: "/packages/cleanup", + tags: ["Packages"], + summary: "Reclaim abandoned local package objects older than one day", + security: [{ Bearer: [] }], + middleware: [requireModerator()] as const, + responses: { + 200: { + description: "Cleanup completed", + content: { + "application/json": { + schema: z.object({ result: z.object({ deleted: z.number() }) }) + } + } + }, + ...packageErrors + } + }), + async (c) => { + try { + await new PackageJobsDatabase(c.env.DB_EXTENSIONS).expire(); + return c.json( + { + result: { + deleted: await cleanupPackageObjects( + new PackageBuildDatabase(c.env.DB_EXTENSIONS), + packageStore(c)! + ) + } + }, + 200 + ); + } catch (error) { + return failure(c, error); + } + } + ); + + for (const resource of ["metadata", "download", "icon"] as const) { + const download = resource === "download"; + const icon = resource === "icon"; + app.openapi( + createRoute({ + method: "get", + path: icon + ? "/packages/{packageId}/icon" + : download + ? "/packages/{packageId}/download" + : "/packages/{packageId}", + tags: ["Packages"], + summary: download + ? "Download an approved package, or preview as owner/moderator" + : "Read an approved package or an authorized private preview", + security: [{ Bearer: [] }], + middleware: [optionalAuth()] as const, + request: { params: PackageParamSchema }, + responses: { + 200: + download || icon + ? { + description: icon + ? "Sanitized SVG, optimized AVIF/PNG, or generic fallback" + : "Verified immutable ZIP", + content: icon + ? { + "image/svg+xml": { + schema: z.string().openapi({ format: "binary" }) + }, + "image/png": { + schema: z.string().openapi({ format: "binary" }) + }, + "image/avif": { + schema: z.string().openapi({ format: "binary" }) + } + } + : { + "application/zip": { + schema: z.string().openapi({ format: "binary" }) + } + } + } + : { + description: "Package manifest and checksum", + content: { + "application/json": { + schema: z.object({ result: PackageArtifactSchema }) + } + } + }, + ...packageErrors + } + }), + async (c: Context<{ Bindings: CloudflareBindings }>) => { + try { + const auth = getOptionalAuth(c); + const row = await new PackageBuildDatabase( + c.env.DB_EXTENSIONS + ).getArtifactForReader(c.req.param("packageId")!, auth?.userId ?? ""); + if (!row) + throw new PackageError("NOT_FOUND", "Package not found", 404); + if (!download && !icon) return c.json(artifactResponse(row), 200); + if (icon) { + const avif = (c.req.header("Accept") ?? "") + .split(",") + .some((part) => { + const [type, ...params] = part.trim().split(";"); + return ( + type === "image/avif" && + !params.some((param) => /^\s*q=0(?:\.0*)?\s*$/.test(param)) + ); + }); + if (!row.icon_source_id) + throw new PackageError( + "ICON_UNAVAILABLE", + "Package icon has not been prepared", + 503 + ); + const key = + row.svg_object_id ?? + (avif ? row.avif_object_id : row.png_object_id); + const checksum = row.svg_object_id + ? row.svg_sha256 + : avif + ? row.avif_sha256 + : row.png_sha256; + const fallback = + ''; + const body = key + ? await packageStore(c)!.get(key, checksum!) + : fallback; + if (!body) + throw new PackageError( + "ICON_UNAVAILABLE", + "Stored icon is unavailable", + 503 + ); + + return new Response(body, { + headers: { + "Content-Type": row.svg_object_id + ? "image/svg+xml" + : key + ? avif + ? "image/avif" + : "image/png" + : "image/svg+xml", + "Content-Security-Policy": + "sandbox; default-src 'none'; style-src 'unsafe-inline'; img-src data:", + "X-Content-Type-Options": "nosniff", + "Cache-Control": "private, no-store", + Vary: "Authorization, Accept" + } + }); + } + const body = await packageStore(c)!.get(row.object_id, row.sha256); + if (!body) + throw new PackageError( + "PACKAGE_UNAVAILABLE", + "Package object is unavailable", + 503 + ); + return new Response(body, { + headers: { + "Content-Type": "application/zip", + "Content-Disposition": `attachment; filename="${row.extension_identifier}-${row.version}-${row.id}.zip"`, + ETag: `"${row.sha256}"`, + "X-Content-Type-Options": "nosniff", + "Cache-Control": "private, no-store", + Vary: "Authorization" + } + }); + } catch (error) { + return failure(c, error); + } + } + ); + } +} diff --git a/src/services/extensions/v2/schemas/badges.ts b/src/services/extensions/v2/schemas/badges.ts index 94a1f715..1d34628a 100644 --- a/src/services/extensions/v2/schemas/badges.ts +++ b/src/services/extensions/v2/schemas/badges.ts @@ -27,6 +27,7 @@ export const BadgeDataSchema = z.object({ releases: z .array( z.object({ + is_latest: z.boolean().optional(), tag: z .string() .min(1) diff --git a/src/services/extensions/v2/schemas/extensions.ts b/src/services/extensions/v2/schemas/extensions.ts index c7e93642..be505f97 100644 --- a/src/services/extensions/v2/schemas/extensions.ts +++ b/src/services/extensions/v2/schemas/extensions.ts @@ -16,6 +16,17 @@ export const ReleaseSchema = z date: z.string().min(1).max(64), download_url: httpUrl(), changelog_url: httpUrl().optional(), + is_latest: z + .boolean() + .optional() + .describe( + "Explicitly selected latest release; at most one release may be selected" + ), + artifact_id: z.uuid().optional(), + sha256: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), requires: ManifestRequirementsSchema }) .strict() @@ -50,7 +61,6 @@ export const ExtensionContentSchema = z releases: z.array(ReleaseSchema).min(1).max(100), homepage_url: ManifestSchema.shape.homepage_url, license: LicenseSchema, - icon_url: ManifestSchema.shape.icon_url, readme: z.string().min(1).max(100_000), source: RepositorySchema, version: ManifestSchema.shape.version, @@ -91,7 +101,7 @@ export const StoredExtensionContentSchema = ExtensionContentSchema.extend({ name: z.string().min(1).max(120), description: z.string().min(1).max(4000), version: z.string().min(1).max(100), - icon_url: z.union([ManifestSchema.shape.icon_url, httpUrl()]).optional(), + icon_url: z.string().max(2048).optional(), website: httpUrl().optional(), license: z.union([ LicenseSchema, diff --git a/src/services/extensions/v2/schemas/manifest.ts b/src/services/extensions/v2/schemas/manifest.ts index ebee1f46..e838dc6c 100644 --- a/src/services/extensions/v2/schemas/manifest.ts +++ b/src/services/extensions/v2/schemas/manifest.ts @@ -2,7 +2,7 @@ import { z } from "@hono/zod-openapi"; import { httpUrl } from "./common"; // Package metadata follows FOSSBilling/modules manifest.schema.json (v1). -// Package paths are kept relative; serving an icon is a publisher concern. +// Icons use fixed filenames at the package root, outside the manifest. export const MANIFEST_EXTENSION_TYPES = [ "module", "theme", @@ -59,12 +59,6 @@ export const ManifestSchema = z .max(1000) .regex(/^[A-Za-z0-9.\-+_:() |]+$/), homepage_url: httpUrl().optional(), - icon_url: z - .string() - .min(1) - .max(2048) - .regex(/^(?!\/)(?![a-zA-Z][a-zA-Z0-9+.-]*:)(?!.*\.\.)[^\\]+$/) - .optional(), requires: ManifestRequirementsSchema }) .strict() diff --git a/src/services/extensions/v2/schemas/packages.ts b/src/services/extensions/v2/schemas/packages.ts new file mode 100644 index 00000000..64f90d40 --- /dev/null +++ b/src/services/extensions/v2/schemas/packages.ts @@ -0,0 +1,52 @@ +import { z } from "@hono/zod-openapi"; +import { ManifestSchema } from "./manifest"; + +export const PackageBuildSchema = z + .object({ + manifest: z + .string() + .min(1) + .max(64 * 1024) + .describe("JSON manifest v1"), + source: z + .instanceof(File) + .optional() + .openapi({ type: "string", format: "binary" }), + source_id: z.uuid().optional() + }) + .strict() + .refine( + (value) => Boolean(value.source) !== Boolean(value.source_id), + "Supply either a source ZIP or a stored source_id" + ); +export const PackageParamSchema = z.object({ packageId: z.uuid() }); +export const PackageArtifactSchema = z + .object({ + id: z.uuid(), + source_id: z.uuid(), + manifest: ManifestSchema, + sha256: z.string().regex(/^[a-f0-9]{64}$/), + download_url: z.url(), + published_at: z.string().nullable(), + created_at: z.string() + }) + .openapi("PackageArtifact"); + +export const PackageJobParamSchema = z.object({ jobId: z.uuid() }); +export const PackageJobSchema = z + .object({ + id: z.uuid(), + extension_identifier: z.string(), + status: z.enum(["queued", "processing", "ready", "failed"]), + artifact: z.union([PackageArtifactSchema, z.null()]), + error: z + .object({ + code: z.string(), + message: z.string(), + status: z.number().int() + }) + .nullable(), + created_at: z.string(), + updated_at: z.string() + }) + .openapi("PackageJob"); diff --git a/test/integration/scheduled.test.ts b/test/integration/scheduled.test.ts new file mode 100644 index 00000000..0a17d647 --- /dev/null +++ b/test/integration/scheduled.test.ts @@ -0,0 +1,33 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { env } from "cloudflare:workers"; +const mocks = vi.hoisted(() => ({ + recover: vi.fn(), + maintain: vi.fn(), + report: vi.fn() +})); +vi.mock("../../src/lib/adapters/cloudflare/package-jobs", () => ({ + consumePackageJobs: vi.fn(), + recoverPackageJobs: mocks.recover +})); +vi.mock("../../src/services/extensions/v2/db/resource-maintenance", () => ({ + maintainExtensionResources: mocks.maintain, + reportExtensionResources: mocks.report +})); +import app from "../../src/app"; +afterEach(() => vi.restoreAllMocks()); +describe("scheduled maintenance", () => { + it("continues resource maintenance when package recovery fails", async () => { + const log = vi.spyOn(console, "error").mockImplementation(() => {}); + mocks.recover.mockRejectedValueOnce(new Error("job database unavailable")); + mocks.maintain.mockResolvedValueOnce(undefined); + mocks.report.mockResolvedValueOnce(undefined); + await app.scheduled({} as ScheduledController, env); + expect(mocks.maintain).toHaveBeenCalledOnce(); + expect(mocks.report).toHaveBeenCalledOnce(); + expect(log).toHaveBeenCalledWith( + expect.objectContaining({ + context: { error: "job database unavailable" } + }) + ); + }); +}); diff --git a/test/lib/auth/bearer-assertion.test.ts b/test/lib/auth/bearer-assertion.test.ts index da218e21..44ef7c4d 100644 --- a/test/lib/auth/bearer-assertion.test.ts +++ b/test/lib/auth/bearer-assertion.test.ts @@ -20,6 +20,9 @@ function platformWithSecret( previousSecret?: string ): PlatformContext { return { + getQueue: () => { + throw new Error("No queue in auth test"); + }, getCache: () => { throw new Error("not implemented"); }, diff --git a/test/lib/context.test.ts b/test/lib/context.test.ts index 3a936758..86d305c9 100644 --- a/test/lib/context.test.ts +++ b/test/lib/context.test.ts @@ -9,6 +9,7 @@ import { IPlatformBindings } from "../../src/lib/interfaces"; // that boots the real app. describe("createPlatformContext", () => { const mockBindings: IPlatformBindings = { + queues: { testQueue: { send: async () => {} } }, caches: { testCache: { get: async () => null, @@ -33,9 +34,16 @@ describe("createPlatformContext", () => { expect(context.getCache("testCache")).toBe(mockBindings.caches.testCache); expect(context.getEnv("TEST_VAR")).toBe("test-value"); + expect(context.getQueue("testQueue")).toBe(mockBindings.queues!.testQueue); expect(context.raw).toBe(mockHonoContext); }); + it("throws when a queue binding is missing", () => { + const context = createPlatformContext(mockHonoContext, mockBindings); + expect(() => context.getQueue("missing")).toThrow( + "Queue binding 'missing' not found" + ); + }); it("throws when cache binding not found", () => { const context = createPlatformContext(mockHonoContext, mockBindings); diff --git a/test/lib/releases.test.ts b/test/lib/releases.test.ts index 58deb2a3..650f43df 100644 --- a/test/lib/releases.test.ts +++ b/test/lib/releases.test.ts @@ -29,3 +29,18 @@ describe("sortReleasesDescending", () => { ]); }); }); + +// The same choice drives directory downloads and v1 version badges. +describe("latestRelease", () => { + it("honors a selected older release and otherwise prefers stable versions", async () => { + const { latestRelease } = await import("../../src/lib/releases"); + const older = { tag: "1.0.0", is_latest: true }; + expect(latestRelease([{ tag: "2.0.0" }, older])).toBe(older); + expect(latestRelease([{ tag: "3.0.0-rc.1" }, { tag: "2.0.0" }])?.tag).toBe( + "2.0.0" + ); + expect( + latestRelease([{ tag: "3.0.0-rc.1" }, { tag: "3.0.0-rc.2" }])?.tag + ).toBe("3.0.0-rc.2"); + }); +}); diff --git a/test/services/extensions/v1/index.test.ts b/test/services/extensions/v1/index.test.ts index ca3834db..41f6adbc 100644 --- a/test/services/extensions/v1/index.test.ts +++ b/test/services/extensions/v1/index.test.ts @@ -411,6 +411,57 @@ describe("Extensions API v1", () => { }); describe("GET /:id/version", () => { + it.each([false, true])( + "uses the selected or stable release when a newer prerelease exists (selected=%s)", + async (selected) => { + const db = getExtensionsDb(env.DB_EXTENSIONS); + const releases = [ + { + tag: "2.0.0-beta.1", + date: "2026-01-02", + download_url: "https://example.test/beta.zip", + requires: { fossbilling: "*" } + }, + { + tag: "1.0.0", + date: "2026-01-01", + download_url: "https://example.test/stable.zip", + requires: { fossbilling: "*" } + }, + { + tag: "0.9.0", + date: "2025-01-01", + download_url: "https://example.test/pinned.zip", + requires: { fossbilling: "*" }, + ...(selected ? { is_latest: true } : {}) + } + ]; + const expected = selected ? "0.9.0" : "1.0.0"; + await db + .update(extensions) + .set({ releases: JSON.stringify(releases), version: expected }) + .where(eq(extensions.identifier, "Example")); + const ctx = createExecutionContext(); + const response = await app.request( + "/extensions/v1/Example/version", + {}, + env, + ctx + ); + await waitOnExecutionContext(ctx); + expect(await response.text()).toBe(expected); + const detail = await app.request( + "/extensions/v1/Example", + {}, + env, + createExecutionContext() + ); + expect( + ((await detail.json()) as { result: { version: string } }).result + .version + ).toBe(expected); + } + ); it("should return plain text version", async () => { const ctx = createExecutionContext(); const res = await app.request( diff --git a/test/services/extensions/v2/b2-store.node.test.ts b/test/services/extensions/v2/b2-store.node.test.ts new file mode 100644 index 00000000..90aedd2f --- /dev/null +++ b/test/services/extensions/v2/b2-store.node.test.ts @@ -0,0 +1,213 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { b2ObjectStore } from "../../../../src/lib/adapters/b2/package-store"; +import { + sha256, + PACKAGE_LIMITS +} from "../../../../src/services/extensions/v2/package-builder"; + +const objectId = "134cf413-b5ab-4f57-b2ad-520020af880d"; +const config: Record = { + ENDPOINT: "https://s3.example.test", + REGION: "eu-central-003", + BUCKET: "private-packages", + ACCESS_KEY_ID: "test-key", + SECRET_ACCESS_KEY: "test-secret" +}; +const store = (overrides: Record = {}) => + b2ObjectStore( + (name) => + ({ ...config, ...overrides })[name.replace("EXTENSION_PACKAGES_S3_", "")], + "packages" + ); +afterEach(() => vi.unstubAllGlobals()); + +describe("B2 object storage", () => { + it("signs ZIP writes and verifies downloaded bytes", async () => { + const bytes = new Uint8Array(new TextEncoder().encode("synthetic ZIP")); + const checksum = await sha256(bytes); + const requests: Request[] = []; + vi.stubGlobal("fetch", async (request: Request) => { + requests.push(request); + if (request.method === "PUT") { + expect(new Uint8Array(await request.arrayBuffer())).toEqual(bytes); + return new Response(null, { status: 200 }); + } + return new Response(bytes, { + headers: { "x-amz-meta-sha256": checksum } + }); + }); + const storage = store(); + await storage.put(objectId, bytes, checksum); + expect( + new Uint8Array( + await new Response(await storage.get(objectId, checksum)).arrayBuffer() + ) + ).toEqual(bytes); + expect(requests.map((r) => r.method)).toEqual(["PUT", "GET"]); + for (const request of requests) { + expect(request.url).toBe( + `https://s3.example.test/private-packages/local/packages/${objectId}` + ); + expect(request.headers.get("Authorization")).toMatch( + /^AWS4-HMAC-SHA256 / + ); + expect(request.redirect).toBe("manual"); + } + expect(requests[0].headers.get("Cache-Control")).toBe("private, no-store"); + expect(requests[0].headers.get("x-amz-content-sha256")).toBe(checksum); + }); + + it.each(["svg", "avif", "png"])( + "stores and cleans up public %s assets under their UUID path", + async (format) => { + const bytes = new Uint8Array(new TextEncoder().encode("prepared image")); + const checksum = await sha256(bytes); + const key = `extensions/${objectId}/icons/${objectId}.${format}`; + const requests: Request[] = []; + vi.stubGlobal("fetch", async (request: Request) => { + requests.push(request); + return new Response(request.method === "GET" ? bytes : null, { + headers: { + "x-amz-meta-sha256": checksum, + "x-amz-version-id": "asset-version" + } + }); + }); + const storage = b2ObjectStore( + (name) => + name === "EXTENSION_ASSETS_PUBLIC_URL" + ? "https://assets.example.test" + : { ...config, BUCKET: "public-assets" }[ + name.replace("EXTENSION_ASSETS_S3_", "") + ], + "assets" + ); + await storage.put(key, bytes, checksum); + expect(await new Response(await storage.get(key, checksum)).text()).toBe( + "prepared image" + ); + await storage.delete(key); + expect(requests.map((r) => r.method)).toEqual([ + "PUT", + "GET", + "HEAD", + "DELETE" + ]); + for (const request of requests) { + expect(new URL(request.url).pathname).toBe(`/public-assets/${key}`); + expect(request.headers.get("Authorization")).toContain( + "AWS4-HMAC-SHA256" + ); + } + expect(requests[0].headers.get("Content-Type")).toBe( + format === "svg" ? "image/svg+xml" : `image/${format}` + ); + expect(requests[0].headers.get("Cache-Control")).toBe( + "public, max-age=31536000, immutable" + ); + expect(new URL(requests[3].url).searchParams.get("versionId")).toBe( + "asset-version" + ); + } + ); + + it("rejects corrupted bodies even when checksum metadata matches", async () => { + const checksum = await sha256( + new Uint8Array(new TextEncoder().encode("expected")) + ); + vi.stubGlobal( + "fetch", + async () => + new Response("corrupted", { + headers: { "x-amz-meta-sha256": checksum } + }) + ); + await expect(store().get(objectId, checksum)).rejects.toThrow( + "checksum verification" + ); + }); + + it("bounds chunked downloads without trusting Content-Length", async () => { + vi.stubGlobal( + "fetch", + async () => + new Response( + new ReadableStream({ + start(controller) { + controller.enqueue( + new Uint8Array(PACKAGE_LIMITS.expandedBytes + 1024 * 1024 + 1) + ); + controller.close(); + } + }) + ) + ); + await expect(store().get(objectId)).rejects.toThrow("size limit"); + }); + + it("deletes the stored version rather than creating a delete marker", async () => { + const requests: Request[] = []; + vi.stubGlobal("fetch", async (request: Request) => { + requests.push(request); + return request.method === "HEAD" + ? new Response(null, { + headers: { "x-amz-version-id": "stored-version+1" } + }) + : new Response(null, { status: 204 }); + }); + await store().delete(objectId); + expect(requests.map((r) => r.method)).toEqual(["HEAD", "DELETE"]); + expect(new URL(requests[1].url).searchParams.get("versionId")).toBe( + "stored-version+1" + ); + }); + + it("treats missing reads and cleanup as absent, but fails closed on storage errors", async () => { + const fetch = vi + .fn() + .mockImplementation(async () => new Response(null, { status: 404 })); + vi.stubGlobal("fetch", fetch); + expect(await store().get(objectId)).toBeNull(); + await store().delete(objectId); + expect(fetch).toHaveBeenCalledTimes(2); + fetch.mockImplementation(async () => new Response(null, { status: 403 })); + await expect(store().get(objectId)).rejects.toThrow("unavailable"); + await expect(store().delete(objectId)).rejects.toThrow("cleanup"); + fetch.mockImplementation(async () => { + throw new Error("upstream timeout"); + }); + await expect(store().get(objectId)).rejects.toThrow("request failed"); + }); + + it("rejects failed uploads and retains cleanup failures for retry", async () => { + const bytes = new Uint8Array(new TextEncoder().encode("ZIP")); + vi.stubGlobal("fetch", async (request: Request) => + request.method === "HEAD" + ? new Response(null, { headers: { "x-amz-version-id": "v1" } }) + : new Response(null, { status: 503 }) + ); + await expect( + store().put(objectId, bytes, await sha256(bytes)) + ).rejects.toThrow("upload failed"); + await expect(store().delete(objectId)).rejects.toThrow("cleanup failed"); + }); + + it("rejects unsafe configuration, keys and mismatched upload checksums before network access", async () => { + const fetch = vi.fn(); + vi.stubGlobal("fetch", fetch); + for (const ENDPOINT of [ + "http://s3.example.test", + "https://key:secret@s3.example.test", + "https://s3.example.test/path", + "https://s3.example.test?key=value" + ]) + expect(() => store({ ENDPOINT })).toThrow("configuration"); + expect(() => store({ SECRET_ACCESS_KEY: undefined })).toThrow("incomplete"); + expect(() => store({ PREFIX: "../other" })).toThrow("configuration"); + await expect(store().get("../other")).rejects.toThrow("object ID"); + await expect( + store().put(objectId, new Uint8Array([1]), "wrong") + ).rejects.toThrow("checksum"); + expect(fetch).not.toHaveBeenCalled(); + }); +}); diff --git a/test/services/extensions/v2/badges.test.ts b/test/services/extensions/v2/badges.test.ts index 2adac488..4995ca4c 100644 --- a/test/services/extensions/v2/badges.test.ts +++ b/test/services/extensions/v2/badges.test.ts @@ -44,6 +44,45 @@ beforeEach(async () => { }); describe("v2 public badges", () => { + it("uses the selected release for badges and the legacy version endpoint", async () => { + await db + .prepare("UPDATE extensions SET releases=? WHERE identifier='example'") + .bind( + JSON.stringify([ + { ...release("1.0.0", "^0.7"), is_latest: true }, + release("2.0.0", requirement) + ]) + ) + .run(); + expect(await (await badge("version")).text()).toContain( + "Latest version: v1.0.0" + ); + expect(await (await badge("fossbilling")).text()).toContain( + "FOSSBilling: ^0.7" + ); + expect(await (await get("/extensions/v1/example/version", {})).text()).toBe( + "1.0.0" + ); + }); + + it("prefers stable releases over newer prereleases when no release is selected", async () => { + await db + .prepare("UPDATE extensions SET releases=? WHERE identifier='example'") + .bind( + JSON.stringify([ + release("3.0.0-beta.1", "^1.0"), + release("2.0.0", requirement) + ]) + ) + .run(); + expect(await (await badge("version")).text()).toContain( + "Latest version: v2.0.0" + ); + expect(await (await get("/extensions/v1/example/version", {})).text()).toBe( + "2.0.0" + ); + }); + it("displays the complete latest-release requirement, including Composer exclusions", async () => { const response = await badge("fossbilling"); expect(response.status).toBe(200); diff --git a/test/services/extensions/v2/db-fixtures.ts b/test/services/extensions/v2/db-fixtures.ts index 717d3b95..c974ce30 100644 --- a/test/services/extensions/v2/db-fixtures.ts +++ b/test/services/extensions/v2/db-fixtures.ts @@ -139,6 +139,25 @@ async function clearDeveloperHistory(db: D1Database): Promise { } export async function resetExtensionsDb(db: D1Database): Promise { + // Test reset only: temporarily lift the immutable-row delete guards. + const guards = [ + ["package_sources", "package_source_immutable"], + ["package_artifacts", "package_artifact_immutable"], + ["package_icons", "package_icons_immutable"] + ]; + await db.batch([ + ...guards.map(([table]) => db.prepare(`DROP TRIGGER ${table}_no_delete`)), + db.prepare("DELETE FROM package_jobs"), + db.prepare("DELETE FROM package_artifacts"), + db.prepare("DELETE FROM package_icons"), + db.prepare("DELETE FROM package_sources"), + db.prepare("DELETE FROM package_objects"), + ...guards.map(([table, message]) => + db.prepare( + `CREATE TRIGGER ${table}_no_delete BEFORE DELETE ON ${table} BEGIN SELECT RAISE(ABORT, '${message}'); END` + ) + ) + ]); await db.prepare("DELETE FROM extension_write_events").run(); await db.prepare("DELETE FROM claim_verification_budgets").run(); await clearDeveloperHistory(db); diff --git a/test/services/extensions/v2/manifest.test.ts b/test/services/extensions/v2/manifest.test.ts index efd8a7f8..9eb547cb 100644 --- a/test/services/extensions/v2/manifest.test.ts +++ b/test/services/extensions/v2/manifest.test.ts @@ -25,7 +25,6 @@ describe("manifest v1 publishing", () => { const content = { ...sampleCreate({ extensionId: "my_module" }), author: { name: "Package Author", url: "https://author.example.com" }, - icon_url: "assets/icon.svg", license: "MIT OR Apache-2.0", requires: { fossbilling: "^0.8.9 || ^1.0", diff --git a/test/services/extensions/v2/package-build.node.test.ts b/test/services/extensions/v2/package-build.node.test.ts new file mode 100644 index 00000000..97806995 --- /dev/null +++ b/test/services/extensions/v2/package-build.node.test.ts @@ -0,0 +1,174 @@ +import { describe, expect, it, vi } from "vitest"; +import { + TextReader, + TextWriter, + Uint8ArrayReader, + Uint8ArrayWriter, + ZipReader, + ZipWriter +} from "@zip.js/zip.js"; +import { prepareQueuedPackage } from "../../../../src/services/extensions/v2/packages/prepare"; +import type { + PackageRepository, + ArtifactRow, + SourceRow, + IconObject +} from "../../../../src/services/extensions/v2/packages/repository"; +import type { PackageJob } from "../../../../src/services/extensions/v2/packages/jobs"; +import type { PackageStore } from "../../../../src/services/extensions/v2/packages/store"; +import { sha256 } from "../../../../src/services/extensions/v2/package-builder"; + +const zipOptions = { useWebWorkers: false, useCompressionStream: true }; +describe("package preparation outside Cloudflare", () => { + it("prepares and corrects ZIPs with injected persistence, storage and image processing", async () => { + const manifest = { + manifest_version: 1 as const, + type: "module" as const, + id: "node-example", + name: "Node Example", + version: "1.0.0", + description: "Prepared in Node", + author: { name: "Developer" }, + license: "MIT", + requires: { fossbilling: ">=0.8.0" } + }; + const owner = { + id: crypto.randomUUID(), + profile_generation: crypto.randomUUID(), + ownership_epoch: 1, + owner_user_id: "owner" + }; + const extensionId = "019a0000-0000-7000-8000-000000000001"; + let source: SourceRow | null = null; + let icons: IconObject[] | null = null; + const artifacts = new Map(); + const unexpected = async (): Promise => { + throw new Error("Unexpected cleanup in normal preparation"); + }; + const repository: PackageRepository = { + getOwner: async () => owner, + findSource: async () => source, + findArtifact: async (_sourceId, hash) => artifacts.get(hash) ?? null, + hasIcon: async () => icons !== null, + reserveObject: async () => {}, + removeObject: unexpected, + claimCleanup: unexpected, + finishCleanup: unexpected, + linkSource: async (_owner, _manifest, input) => (source ??= input), + ensureDraft: async () => extensionId, + linkIcon: async (_owner, _identifier, _sourceId, input) => { + icons = input; + return true; + }, + linkArtifact: async (_owner, metadata, input) => { + const svg = icons?.find((icon) => icon.format === "svg"); + const row: ArtifactRow = { + id: input.id, + source_id: input.sourceId, + manifest: JSON.stringify(metadata), + sha256: input.sha256, + object_id: input.objectId, + download_url: input.downloadUrl, + published_at: null, + created_at: new Date().toISOString(), + extension_identifier: metadata.id, + profile_generation: owner.profile_generation, + version: metadata.version, + icon_source_id: input.sourceId, + svg_object_id: svg?.id ?? null, + svg_sha256: svg?.checksum ?? null, + avif_object_id: null, + avif_sha256: null, + png_object_id: null, + png_sha256: null + }; + artifacts.set(input.manifestHash, row); + return row; + } + }; + const objects = new Map>(); + const store: PackageStore = { + put: async (key, bytes) => { + objects.set(key, new Uint8Array(bytes)); + }, + get: async (key) => + objects.has(key) ? new Blob([objects.get(key)!]).stream() : null, + delete: async (key) => { + objects.delete(key); + } + }; + const images = { + prepare: vi.fn(async (bytes: Uint8Array) => [ + { format: "svg" as const, bytes } + ]) + }; + const writer = new ZipWriter(new Uint8ArrayWriter(), zipOptions); + await writer.add( + "Service.php", + new TextReader("' + ) + ); + const upload = await writer.close(); + const uploadId = crypto.randomUUID(); + objects.set(uploadId, upload); + const job: PackageJob = { + id: crypto.randomUUID(), + fingerprint: "node", + owner, + manifest, + sourceId: null, + uploadObjectId: uploadId, + uploadSha256: await sha256(upload), + status: "processing", + attempts: 1, + artifactId: null, + error: null, + createdAt: "now", + updatedAt: "now" + }; + const services = { + repository, + store, + images, + baseUrl: "https://directory.example.test/extensions/v2" + }; + const id = await prepareQueuedPackage(services, job); + const first = [...artifacts.values()].find((row) => row.id === id)!; + const bytes = objects.get(first.object_id)!; + expect(await sha256(bytes)).toBe(first.sha256); + const reader = new ZipReader(new Uint8ArrayReader(bytes), zipOptions); + try { + const files: Record = {}; + for (const entry of await reader.getEntries()) { + if (!entry.directory) + files[entry.filename] = await entry.getData!(new TextWriter()); + } + expect(JSON.parse(files["manifest.json"])).toEqual(manifest); + expect(files["Service.php"]).toBe(" row.id === correctedId + )!; + expect(corrected.source_id).toBe(first.source_id); + expect(corrected.svg_object_id).toBe(first.svg_object_id); + expect(corrected.sha256).not.toBe(first.sha256); + expect(images.prepare).toHaveBeenCalledOnce(); + }); +}); diff --git a/test/services/extensions/v2/package-builder.test.ts b/test/services/extensions/v2/package-builder.test.ts new file mode 100644 index 00000000..e3dc87a8 --- /dev/null +++ b/test/services/extensions/v2/package-builder.test.ts @@ -0,0 +1,322 @@ +import { describe, expect, it } from "vitest"; +import { + TextReader, + TextWriter, + Uint8ArrayReader, + Uint8ArrayWriter, + ZipReader, + ZipWriter +} from "@zip.js/zip.js"; +import { + buildExtensionPackage, + PACKAGE_LIMITS, + sha256 +} from "../../../../src/services/extensions/v2/package-builder"; + +const manifest = { + manifest_version: 1, + type: "module", + id: "demo", + name: "Demo", + version: "1.2.0", + description: "Example extension", + author: { name: "Example Developer" }, + license: "Apache-2.0", + requires: { fossbilling: ">=0.8.0" } +}; +const options = { useWebWorkers: false, useCompressionStream: true }; +async function zip( + files: [string, string][], + entryOptions: Parameters>["add"]>[2] = {} +) { + const writer = new ZipWriter(new Uint8ArrayWriter(), options); + for (const [name, content] of files) { + await writer.add(name, new TextReader(content), entryOptions); + } + return writer.close(); +} +async function contents(bytes: Uint8Array) { + const reader = new ZipReader(new Uint8ArrayReader(bytes), options); + try { + const result: Record = {}; + for await (const entry of reader.getEntriesGenerator()) { + if (!entry.directory) + result[entry.filename] = await entry.getData(new TextWriter()); + } + return result; + } finally { + await reader.close(); + } +} + +describe("directory package builder", () => { + it("rejects overlong and deeply nested paths before walking ancestors", async () => { + for (const path of [ + "a/".repeat(20000) + "file.php", + "a/".repeat(PACKAGE_LIMITS.pathDepth) + "file.php", + "a".repeat(PACKAGE_LIMITS.pathLength + 1) + ]) { + await expect( + buildExtensionPackage(await zip([[path, "file"]]), manifest) + ).rejects.toThrow(/path exceeds/); + } + const path = "a/".repeat(PACKAGE_LIMITS.pathDepth - 1) + "file.php"; + await expect( + buildExtensionPackage(await zip([[path, "file"]]), manifest) + ).resolves.toMatchObject({ manifest }); + }); + it("replaces the manifest and preserves every source file", async () => { + const source = await zip([ + ["manifest.json", "old metadata"], + ["Service.php", ""] + ]); + const built = await buildExtensionPackage(source, manifest); + expect(await contents(built.bytes)).toEqual({ + "manifest.json": JSON.stringify(manifest, null, 2) + "\n", + "Service.php": "" + }); + expect(built.sourceSha256).toBe(await sha256(source)); + expect(built.packageSha256).toBe(await sha256(built.bytes)); + const repeated = await buildExtensionPackage(source, manifest); + expect(repeated.packageSha256).toBe(built.packageSha256); + const correction = await buildExtensionPackage(source, { + ...manifest, + description: "Corrected description" + }); + expect(correction.sourceSha256).toBe(built.sourceSha256); + expect(correction.manifest.version).toBe(built.manifest.version); + expect(correction.packageSha256).not.toBe(built.packageSha256); + }); + + it.each([0, 8])( + "preserves the exact compressed payload for method %i", + async (compressionMethod) => { + const source = await zip( + [["file.php", ") { + const reader = new ZipReader(new Uint8ArrayReader(bytes), options); + try { + const entries = await reader.getEntries(); + const entry = entries.find((entry) => entry.filename === "file.php")!; + if (!entry || entry.directory) + throw new Error("Expected a regular file"); + return { + method: entry.compressionMethod, + bytes: await entry.getData(new Uint8ArrayWriter(), { + passThrough: true + }) + }; + } finally { + await reader.close(); + } + } + const original = await payload(source); + expect(original.method).toBe(compressionMethod); + expect(await payload(built.bytes)).toEqual(original); + expect((await contents(built.bytes))["file.php"]).toBe( + " { + const source = await zip([ + [`icon.${format}`, "image bytes"], + ["file.php", " { + const source = await zip([ + ["icon.svg", "svg"], + ["icon.png", "png"] + ]); + await expect(buildExtensionPackage(source, manifest)).rejects.toThrow( + "only one" + ); + await expect( + buildExtensionPackage(await zip([["file.php", " { + const built = await buildExtensionPackage( + await zip([["assets/icon.svg", "image"]]), + manifest + ); + expect(built.icon).toBeNull(); + }); + + it("inserts a missing manifest and rejects invalid metadata", async () => { + const source = await zip([["file.php", " { + await expect( + buildExtensionPackage(await zip([[path, "test"]]), manifest) + ).rejects.toThrow(/Unsafe/); + }); + + it("rejects case collisions, file/directory conflicts and symlinks", async () => { + await expect( + buildExtensionPackage( + await zip([ + ["A.php", "a"], + ["a.php", "b"] + ]), + manifest + ) + ).rejects.toThrow("Duplicate package path"); + await expect( + buildExtensionPackage( + await zip([ + ["dir", "a"], + ["dir/file.php", "b"] + ]), + manifest + ) + ).rejects.toThrow("File conflicts"); + await expect( + buildExtensionPackage( + await zip([["link", "../outside"]], { unixMode: 0o120777 }), + manifest + ) + ).rejects.toThrow("Unsupported package entry"); + }); + + it("rejects files beneath the reserved manifest path", async () => { + await expect( + buildExtensionPackage( + await zip([["manifest.json/file.php", "bad"]]), + manifest + ) + ).rejects.toThrow("Manifest path is a directory"); + }); + + it("rejects encrypted files and truncated archives", async () => { + await expect( + buildExtensionPackage( + await zip([["file", "secret"]], { password: "test" }), + manifest + ) + ).rejects.toThrow("Unsupported package entry"); + const source = await zip([["file", "content"]]); + await expect( + buildExtensionPackage(source.slice(0, -30), manifest) + ).rejects.toThrow(); + }); + + it.each(["manifest.json", "file.php"])( + "checks CRCs before copying or replacing %s", + async (filename) => { + const source = await zip( + [ + [filename, "unique-contents"], + ["Service.php", " + contents.every((byte, part) => source[index + part] === byte) + ); + expect(offset).toBeGreaterThan(0); + source[offset] ^= 1; + await expect(buildExtensionPackage(source, manifest)).rejects.toThrow( + "Invalid CRC32" + ); + } + ); + + it.each([-1, 1])( + "rejects inconsistent uncompressed sizes (%i)", + async (difference) => { + const source = await zip([["file.php", "unique-contents"]]); + const view = new DataView( + source.buffer, + source.byteOffset, + source.byteLength + ); + view.setUint32(22, "unique-contents".length + difference, true); + for (let offset = 0; offset < source.length - 46; offset++) { + if (view.getUint32(offset, true) === 0x08074b50) { + view.setUint32( + offset + 12, + "unique-contents".length + difference, + true + ); + } + if (view.getUint32(offset, true) === 0x02014b50) { + view.setUint32( + offset + 24, + "unique-contents".length + difference, + true + ); + break; + } + } + await expect(buildExtensionPackage(source, manifest)).rejects.toThrow(); + } + ); + + it("rejects oversized inputs and expansion bombs", async () => { + await expect( + buildExtensionPackage( + new Uint8Array(PACKAGE_LIMITS.sourceBytes + 1), + manifest + ) + ).rejects.toThrow("Source ZIP exceeds size limit"); + const source = await zip([ + ["large.txt", "x".repeat(PACKAGE_LIMITS.expandedBytes + 1)] + ]); + expect(source.byteLength).toBeLessThan(PACKAGE_LIMITS.sourceBytes); + await expect(buildExtensionPackage(source, manifest)).rejects.toThrow( + "expanded size limit" + ); + }); + + it("rejects entry floods and manifest-only archives", async () => { + const source = await zip( + Array.from({ length: PACKAGE_LIMITS.entries + 1 }, (_, i) => [ + `file-${i}`, + "" + ]) + ); + await expect(buildExtensionPackage(source, manifest)).rejects.toThrow( + "entry limit" + ); + await expect( + buildExtensionPackage(await zip([["manifest.json", "{}"]]), manifest) + ).rejects.toThrow("no source files"); + }); +}); diff --git a/test/services/extensions/v2/package-jobs.node.test.ts b/test/services/extensions/v2/package-jobs.node.test.ts new file mode 100644 index 00000000..818706a3 --- /dev/null +++ b/test/services/extensions/v2/package-jobs.node.test.ts @@ -0,0 +1,141 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + dispatchPackageJob, + processPackageJob, + type PackageJob, + type PackageJobRepository +} from "../../../../src/services/extensions/v2/packages/jobs"; +import { PackageError } from "../../../../src/services/extensions/v2/packages/errors"; + +function repository() { + const job: PackageJob = { + id: crypto.randomUUID(), + fingerprint: "test", + owner: { + id: "developer", + profile_generation: "generation", + ownership_epoch: 1, + owner_user_id: "owner" + }, + manifest: { + manifest_version: 1, + type: "module", + id: "example", + name: "Example", + version: "1.0.0", + description: "Example", + author: { name: "Developer" }, + license: "MIT", + requires: { fossbilling: "*" } + }, + sourceId: null, + uploadObjectId: "upload", + uploadSha256: "checksum", + status: "queued", + attempts: 0, + artifactId: null, + error: null, + createdAt: "now", + updatedAt: "now" + }; + let activeLease: string | null = null; + const jobs: PackageJobRepository = { + get: async () => ({ ...job }), + claim: async (_id, lease) => { + if (job.status !== "queued") return null; + activeLease = lease; + job.status = "processing"; + job.attempts++; + return { ...job }; + }, + complete: async (_id, lease, artifactId) => { + if (lease === activeLease) { + job.status = "ready"; + job.artifactId = artifactId; + activeLease = null; + } + }, + fail: async (_id, lease, error, retry) => { + if (lease === activeLease) { + job.status = retry ? "queued" : "failed"; + job.error = error; + activeLease = null; + } + }, + pending: async () => [job.id], + markEnqueued: vi.fn().mockResolvedValue(undefined) + }; + return { job, jobs }; +} + +afterEach(() => vi.restoreAllMocks()); + +describe("provider-independent package job processing", () => { + it("publishes only an identifier through the queue interface", async () => { + const { jobs, job } = repository(); + const send = vi.fn().mockResolvedValue(undefined); + await dispatchPackageJob(jobs, { send }, job.id); + expect(send).toHaveBeenCalledWith({ jobId: job.id }); + expect(jobs.markEnqueued).toHaveBeenCalledWith(job.id); + }); + it("leaves an actively leased job unchanged when another delivery arrives", async () => { + const { jobs, job } = repository(); + job.status = "processing"; + const prepare = vi.fn(); + expect(await processPackageJob(jobs, job.id, prepare)).toBe("busy"); + expect(job.status).toBe("processing"); + expect(job.attempts).toBe(0); + expect(prepare).not.toHaveBeenCalled(); + }); + it("terminates jobs interrupted beyond the processing attempt limit", async () => { + const { jobs, job } = repository(); + job.attempts = 3; + const prepare = vi.fn(); + expect(await processPackageJob(jobs, job.id, prepare)).toBe("ack"); + expect(job.attempts).toBe(4); + expect(job.status).toBe("failed"); + expect(job.error).toMatchObject({ + code: "PACKAGE_UNAVAILABLE", + status: 503 + }); + expect(prepare).not.toHaveBeenCalled(); + }); + it("does not run completed jobs twice", async () => { + const { jobs, job } = repository(); + const prepare = vi.fn().mockResolvedValue("artifact"); + expect(await processPackageJob(jobs, job.id, prepare)).toBe("ack"); + expect(await processPackageJob(jobs, job.id, prepare)).toBe("ack"); + expect(prepare).toHaveBeenCalledTimes(1); + expect(job.artifactId).toBe("artifact"); + }); + it("retries transient failures with a bounded attempt count", async () => { + const { jobs, job } = repository(); + const logged = vi.spyOn(console, "error").mockImplementation(() => {}); + const prepare = vi.fn().mockRejectedValue(new Error("storage unavailable")); + expect(await processPackageJob(jobs, job.id, prepare)).toBe("retry"); + expect(await processPackageJob(jobs, job.id, prepare)).toBe("retry"); + expect(await processPackageJob(jobs, job.id, prepare)).toBe("ack"); + expect(job.status).toBe("failed"); + expect(job.error?.message).not.toContain("storage unavailable"); + expect(logged).toHaveBeenCalledWith( + expect.objectContaining({ + context: expect.objectContaining({ + jobId: job.id, + error: "storage unavailable" + }) + }) + ); + }); + it("does not retry invalid packages or revoked ownership", async () => { + for (const status of [422, 403] as const) { + const { jobs, job } = repository(); + expect( + await processPackageJob(jobs, job.id, async () => { + throw new PackageError("REJECTED", "Rejected", status); + }) + ).toBe("ack"); + expect(job.status).toBe("failed"); + expect(job.attempts).toBe(1); + } + }); +}); diff --git a/test/services/extensions/v2/packages.test.ts b/test/services/extensions/v2/packages.test.ts new file mode 100644 index 00000000..8db2efeb --- /dev/null +++ b/test/services/extensions/v2/packages.test.ts @@ -0,0 +1,1721 @@ +import { PackageJobsDatabase } from "../../../../src/services/extensions/v2/db/package-jobs"; +import { processPackageJob } from "../../../../src/services/extensions/v2/packages/jobs"; +import { + recoverPackageJobs, + packagePreparation +} from "../../../../src/lib/adapters/cloudflare/package-jobs"; +import { prepareQueuedPackage } from "../../../../src/services/extensions/v2/packages/prepare"; +import { beforeEach, afterEach, describe, expect, it, vi } from "vitest"; +import { env } from "cloudflare:workers"; +import { + createExecutionContext, + createMessageBatch, + getQueueResult, + waitOnExecutionContext +} from "cloudflare:test"; +import { + TextReader, + Uint8ArrayReader, + Uint8ArrayWriter, + ZipWriter, + ZipReader, + TextWriter +} from "@zip.js/zip.js"; +import app from "../../../../src/app"; +import { + signAccessToken, + activateM2mJwks, + deactivateM2mJwks +} from "../../../lib/auth/m2m-helper"; +import { PackageArtifactSchema } from "../../../../src/services/extensions/v2/schemas/packages"; +import { ManifestSchema } from "../../../../src/services/extensions/v2/schemas/manifest"; +import { ExtensionContentSchema } from "../../../../src/services/extensions/v2/schemas/extensions"; +import { contentManifest } from "../../../../src/services/extensions/v2/packages/manifest"; +import { sha256 } from "../../../../src/services/extensions/v2/package-builder"; +import { + setupExtensionsV2Tests, + authHeaders, + seedDeveloper, + sampleContent, + post, + get, + put, + db +} from "./harness"; +import { insertUser, getExtension, getDeveloper, UUID_V7 } from "./db-fixtures"; + +vi.mock("@octokit/request", async () => + (await import("../../../mocks/octokit")).octokitRequestMock() +); +setupExtensionsV2Tests(); +const options = { useWebWorkers: false, useCompressionStream: true }; +const root = "/extensions/v2"; +const manifest = () => + contentManifest( + "package-demo", + ExtensionContentSchema.parse(sampleContent()) + ); +const packageKey = (id: string) => `/test-packages/local/packages/${id}`; +let b2: ReturnType; + +function mockB2() { + Object.assign(env, { + EXTENSION_PACKAGES_S3_ENDPOINT: "https://s3.example.test", + EXTENSION_PACKAGES_S3_REGION: "test-region", + EXTENSION_PACKAGES_S3_BUCKET: "test-packages", + EXTENSION_PACKAGES_S3_PREFIX: "local", + EXTENSION_PACKAGES_S3_ACCESS_KEY_ID: "test-key", + EXTENSION_PACKAGES_S3_SECRET_ACCESS_KEY: "test-secret", + EXTENSION_ASSETS_S3_ENDPOINT: "https://s3.example.test", + EXTENSION_ASSETS_S3_REGION: "test-region", + EXTENSION_ASSETS_S3_BUCKET: "test-assets", + EXTENSION_ASSETS_S3_ACCESS_KEY_ID: "test-key", + EXTENSION_ASSETS_S3_SECRET_ACCESS_KEY: "test-secret", + EXTENSION_ASSETS_PUBLIC_URL: "https://extensions-assets.fossbilling.net" + }); + const calls: Request[] = []; + const objects = new Map< + string, + { + bytes: ArrayBuffer; + checksum: string; + version: string; + } + >(); + const fetch = vi.fn(async (request: Request) => { + const url = new URL(request.url); + calls.push(request.clone()); + const key = url.pathname; + if (request.method === "PUT") { + objects.set(key, { + bytes: await request.arrayBuffer(), + checksum: request.headers.get("x-amz-meta-sha256")!, + version: crypto.randomUUID() + }); + return new Response(null); + } + const object = objects.get(key); + if (!object) return new Response(null, { status: 404 }); + if (request.method === "DELETE") { + if (url.searchParams.get("versionId") !== object.version) + return new Response(null, { status: 400 }); + objects.delete(key); + return new Response(null, { status: 204 }); + } + if (!["GET", "HEAD"].includes(request.method)) + return new Response(null, { status: 405 }); + return new Response(request.method === "HEAD" ? null : object.bytes, { + headers: { + "x-amz-meta-sha256": object.checksum, + "x-amz-version-id": object.version + } + }); + }); + vi.stubGlobal("fetch", fetch); + return { calls, objects, fetch }; +} + +beforeEach(async () => { + vi.spyOn(env.EXTENSION_PACKAGE_QUEUE, "send").mockResolvedValue({ + metadata: { metrics: { backlogCount: 0, backlogBytes: 0 } } + }); + b2 = mockB2(); + Object.assign(env, { + ENVIRONMENT: "local", + EXTENSION_PACKAGES_MODE: "local", + EXTENSION_PACKAGES_BASE_URL: "http://localhost:8789/extensions/v2" + }); + await authHeaders("owner"); + await seedDeveloper("developer", "owner"); +}); +afterEach(() => { + const requests = b2.fetch.mock.calls.map(([request]) => request); + vi.restoreAllMocks(); + vi.unstubAllGlobals(); + for (const key of [ + "S3_ENDPOINT", + "S3_REGION", + "S3_BUCKET", + "S3_ACCESS_KEY_ID", + "S3_SECRET_ACCESS_KEY", + "PUBLIC_URL" + ]) + Reflect.deleteProperty(env, `EXTENSION_ASSETS_${key}`); + for (const suffix of [ + "ENDPOINT", + "REGION", + "BUCKET", + "ACCESS_KEY_ID", + "SECRET_ACCESS_KEY", + "PREFIX" + ]) + Reflect.deleteProperty(env, `EXTENSION_PACKAGES_S3_${suffix}`); + env.EXTENSION_PACKAGES_MODE = "disabled"; + // Assert outside the adapter's catch, including calls with injected failures. + for (const request of requests) { + const url = new URL(request.url); + expect(url.hostname).toBe("s3.example.test"); + expect(request.headers.get("Authorization")).toMatch(/^AWS4-HMAC-SHA256 /); + expect(["PUT", "GET", "HEAD", "DELETE"]).toContain(request.method); + if (request.method === "DELETE") + expect(url.searchParams.get("versionId")).toBeTruthy(); + } +}); +async function zip(content = ", + sourceId?: string, + user = "owner", + process = true +) { + const headers = await authHeaders(user); + delete headers["Content-Type"]; + const form = new FormData(); + form.set("manifest", JSON.stringify(metadata)); + if (sourceId) form.set("source_id", sourceId); + else + form.set( + "source", + new File([source ?? (await zip())], "source.zip", { + type: "application/zip" + }) + ); + const ctx = createExecutionContext(); + const response = await app.request( + `${root}/packages`, + { method: "POST", headers, body: form }, + env, + ctx + ); + await waitOnExecutionContext(ctx); + return process ? finishJob(response) : response; +} +async function finishJob(response: Response): Promise { + if (response.status !== 202) return response; + const { result } = (await response.json()) as { result: { id: string } }; + const jobs = new PackageJobsDatabase(env.DB_EXTENSIONS); + let job = (await jobs.get(result.id))!; + for ( + let attempt = 0; + attempt < 100 && job.status !== "ready" && job.status !== "failed"; + attempt++ + ) { + await processPackageJob(jobs, result.id, (job) => + prepareQueuedPackage(packagePreparation(env), job) + ); + job = (await jobs.get(result.id))!; + if (job.status === "processing") + await new Promise((resolve) => setTimeout(resolve, 5)); + } + if (job.status === "ready") { + return get( + `${root}/packages/${job.artifactId}`, + await authHeaders("owner") + ); + } + return Response.json( + { error: job.error }, + { status: job.error?.status ?? 503 } + ); +} +async function built( + metadata: unknown = manifest(), + source?: Uint8Array, + sourceId?: string +) { + const response = await upload(metadata, source, sourceId); + const body = (await response.json()) as { result: unknown }; + expect(response.status, JSON.stringify(body)).toBe(200); + return PackageArtifactSchema.parse(body.result); +} +function content(artifact: ReturnType) { + const { id: _id, ...metadata } = artifact.manifest; + return { + ...sampleContent(), + ...metadata, + download_url: artifact.download_url, + releases: [ + { + tag: artifact.manifest.version, + date: "2026-11-05", + requires: artifact.manifest.requires, + artifact_id: artifact.id, + sha256: artifact.sha256, + download_url: artifact.download_url + } + ] + }; +} +async function submitted( + artifact: ReturnType +) { + const response = await post( + `${root}/extensions`, + await authHeaders("owner"), + { identifier: "package-demo", ...content(artifact) } + ); + const body = (await response.json()) as { result: { revision_id: string } }; + expect(response.status, JSON.stringify(body)).toBe(201); + return body.result.revision_id; +} +async function approve(revision: string) { + await insertUser(db, { id: "moderator", is_moderator: 1 }); + return post( + `${root}/extensions/package-demo/revisions/${revision}/approve`, + await authHeaders("moderator"), + {} + ); +} +async function unpack(bytes: Uint8Array) { + const reader = new ZipReader(new Uint8ArrayReader(bytes), options); + const result: Record = {}; + try { + for await (const entry of reader.getEntriesGenerator()) + if (!entry.directory) + result[entry.filename] = await entry.getData(new TextWriter()); + } finally { + await reader.close(); + } + return result; +} + +describe("publisher package builds", () => { + beforeEach(async () => { + await activateM2mJwks(); + const jwksFetch = globalThis.fetch; + vi.stubGlobal("fetch", (input: RequestInfo | URL, init?: RequestInit) => { + const request = + input instanceof Request ? input : new Request(input, init); + return new URL(request.url).hostname === "s3.example.test" + ? b2.fetch(request) + : jwksFetch(input, init); + }); + await insertUser(db, { id: "publisher-moderator", is_moderator: 1 }); + const response = await put( + `${root}/m2m/clients/package-publisher`, + await authHeaders("publisher-moderator"), + { + developer_id: (await getDeveloper(db, "developer"))!.id, + can_publish: true + } + ); + expect(response.status).toBe(200); + }); + afterEach(() => deactivateM2mJwks()); + async function machineBuild(scopes = ["extensions:push"], process = true) { + const form = new FormData(); + form.set("manifest", JSON.stringify(manifest())); + form.set("source", new File([await zip()], "source.zip")); + const response = await app.request( + `${root}/packages`, + { + method: "POST", + headers: { + Authorization: `Bearer ${await signAccessToken({ clientId: "package-publisher", scopes })}` + }, + body: form + }, + env + ); + return process ? finishJob(response) : response; + } + it("lets an active machine publisher poll its own queued and ready job", async () => { + const response = await machineBuild(["extensions:push"], false); + expect(response.status).toBe(202); + const { result: job } = (await response.json()) as { + result: { id: string }; + }; + const headers = { + Authorization: `Bearer ${await signAccessToken({ clientId: "package-publisher", scopes: ["extensions:push"] })}` + }; + const pending = await get(`${root}/package-jobs/${job.id}`, headers); + expect(pending.status).toBe(200); + expect(await pending.json()).toMatchObject({ + result: { status: "queued", artifact: null } + }); + expect( + (await finishJob(Response.json({ result: job }, { status: 202 }))).status + ).toBe(200); + const ready = await get(`${root}/package-jobs/${job.id}`, headers); + expect(ready.status).toBe(200); + expect(await ready.json()).toMatchObject({ + result: { + status: "ready", + artifact: { manifest: { id: "package-demo" } } + } + }); + await db + .prepare( + "UPDATE m2m_publisher_clients SET revoked_at=CURRENT_TIMESTAMP WHERE client_id='package-publisher'" + ) + .run(); + expect((await get(`${root}/package-jobs/${job.id}`, headers)).status).toBe( + 403 + ); + }); + it("builds under the mapped developer while retaining owner privacy", async () => { + const response = await machineBuild(); + expect(response.status).toBe(200); + const artifact = PackageArtifactSchema.parse( + ((await response.json()) as { result: unknown }).result + ); + const source = await db + .prepare("SELECT profile_generation FROM package_sources WHERE id=?") + .bind(artifact.source_id) + .first(); + expect(source).toBeTruthy(); + expect( + (await get(`${root}/packages/${artifact.id}/download`, {})).status + ).toBe(404); + expect( + ( + await get( + `${root}/packages/${artifact.id}/download`, + await authHeaders("owner") + ) + ).status + ).toBe(200); + const token = await signAccessToken({ + clientId: "package-publisher", + scopes: ["extensions:push", "extensions:publish"] + }); + const headers = { + Authorization: `Bearer ${token}`, + "Content-Type": "application/json" + }; + const created = await post(`${root}/extensions`, headers, { + identifier: "package-demo", + ...content(artifact) + }); + expect(created.status).toBe(201); + const revision = ( + (await created.json()) as { result: { revision_id: string } } + ).result.revision_id; + expect( + ( + await post( + `${root}/extensions/package-demo/revisions/${revision}/approve`, + headers, + {} + ) + ).status + ).toBe(200); + expect( + (await get(`${root}/packages/${artifact.id}/download`, {})).status + ).toBe(200); + }); + it("requires push scope and refuses revoked or stale bindings", async () => { + expect((await machineBuild(["extensions:publish"])).status).toBe(403); + await db + .prepare( + "UPDATE m2m_publisher_clients SET revoked_at=CURRENT_TIMESTAMP WHERE client_id='package-publisher'" + ) + .run(); + expect((await machineBuild()).status).toBe(403); + await db + .prepare( + "UPDATE m2m_publisher_clients SET revoked_at=NULL,ownership_epoch=ownership_epoch+1 WHERE client_id='package-publisher'" + ) + .run(); + expect((await machineBuild()).status).toBe(403); + }); + it("does not reserve a package belonging to a different developer", async () => { + await seedDeveloper("another-developer", "another-owner"); + await db + .prepare( + "UPDATE m2m_publisher_clients SET developer_id=(SELECT id FROM developers WHERE identifier='another-developer') WHERE client_id='package-publisher'" + ) + .run(); + // New IDs can be reserved by that developer, but an existing owned ID cannot. + const artifact = await built(); + await submitted(artifact); + expect((await machineBuild()).status).toBe(403); + }); + it("rechecks revocation after uploading before linking source bytes", async () => { + const original = b2.fetch.getMockImplementation()!; + b2.fetch.mockImplementationOnce(async (request) => { + const result = await original(request); + await db + .prepare( + "UPDATE m2m_publisher_clients SET revoked_at=CURRENT_TIMESTAMP WHERE client_id='package-publisher'" + ) + .run(); + return result; + }); + expect((await machineBuild()).status).toBe(403); + expect( + await db.prepare("SELECT count(*) as count FROM package_sources").first() + ).toEqual({ count: 0 }); + }); +}); + +describe("queued package preparation", () => { + it("accepts a private job without preparing an artifact and deduplicates retries", async () => { + const source = await zip(); + const response = await upload( + manifest(), + source, + undefined, + "owner", + false + ); + expect(response.status).toBe(202); + const { result: job } = (await response.json()) as { + result: { id: string; status: string }; + }; + expect(job.status).toBe("queued"); + expect( + await db + .prepare("SELECT count(*) AS count FROM package_artifacts") + .first() + ).toEqual({ count: 0 }); + expect(env.EXTENSION_PACKAGE_QUEUE.send).toHaveBeenCalledWith( + { jobId: job.id }, + { contentType: "json" } + ); + const repeat = await upload(manifest(), source, undefined, "owner", false); + expect( + ((await repeat.json()) as { result: { id: string } }).result.id + ).toBe(job.id); + expect((await get(`${root}/package-jobs/${job.id}`, {})).status).toBe(401); + expect( + ( + await get( + `${root}/package-jobs/${job.id}`, + await authHeaders("stranger") + ) + ).status + ).toBe(403); + const ctx = createExecutionContext(); + const batch = createMessageBatch("extension-packages", [ + { + id: "delivery", + timestamp: new Date(), + attempts: 1, + body: { jobId: job.id } + } + ]); + await app.queue(batch, env); + expect(await getQueueResult(batch, ctx)).toMatchObject({ + explicitAcks: ["delivery"], + retryMessages: [] + }); + const status = await get( + `${root}/package-jobs/${job.id}`, + await authHeaders("owner") + ); + expect( + ((await status.json()) as { result: { status: string } }).result.status + ).toBe("ready"); + }); + it("does not process a job after its developer ownership changes", async () => { + const response = await upload( + manifest(), + undefined, + undefined, + "owner", + false + ); + const { result: job } = (await response.json()) as { + result: { id: string }; + }; + await db + .prepare( + "UPDATE developers SET ownership_epoch=ownership_epoch+1 WHERE identifier = 'developer'" + ) + .run(); + expect( + (await finishJob(Response.json({ result: job }, { status: 202 }))).status + ).toBe(403); + expect( + await db + .prepare("SELECT count(*) AS count FROM package_artifacts") + .first() + ).toEqual({ count: 0 }); + }); + it("keeps the job recoverable if queue publication fails", async () => { + vi.mocked(env.EXTENSION_PACKAGE_QUEUE.send).mockRejectedValueOnce( + new Error("queue unavailable") + ); + expect( + (await upload(manifest(), undefined, undefined, "owner", false)).status + ).toBe(503); + const jobs = new PackageJobsDatabase(db); + const ids = await jobs.pending(); + expect(ids).toHaveLength(1); + const { dispatchPackageJob } = + await import("../../../../src/services/extensions/v2/packages/jobs"); + const send = vi.fn().mockResolvedValue(undefined); + await dispatchPackageJob(jobs, { send }, ids[0]); + expect(send).toHaveBeenCalledWith({ jobId: ids[0] }); + expect(await jobs.pending()).toEqual([]); + }); + it("continues recovery after one queue dispatch fails", async () => { + await upload(manifest(), undefined, undefined, "owner", false); + await upload( + { ...manifest(), version: "2.0.0" }, + undefined, + undefined, + "owner", + false + ); + await db.prepare("UPDATE package_jobs SET enqueued_at=NULL").run(); + const logged = vi.spyOn(console, "error").mockImplementation(() => {}); + vi.mocked(env.EXTENSION_PACKAGE_QUEUE.send) + .mockClear() + .mockRejectedValueOnce(new Error("queue unavailable")); + await recoverPackageJobs(env); + expect(env.EXTENSION_PACKAGE_QUEUE.send).toHaveBeenCalledTimes(2); + expect(await new PackageJobsDatabase(db).pending()).toHaveLength(1); + expect(logged).toHaveBeenCalledWith( + expect.objectContaining({ + context: expect.objectContaining({ error: "queue unavailable" }) + }) + ); + }); + it("retries a missing staged upload and completes when storage recovers", async () => { + const source = await zip(); + const response = await upload( + manifest(), + source, + undefined, + "owner", + false + ); + const { result } = (await response.json()) as { result: { id: string } }; + const jobs = new PackageJobsDatabase(db); + const job = (await jobs.get(result.id))!; + const key = packageKey(job.uploadObjectId!); + const staged = b2.objects.get(key)!; + b2.objects.delete(key); + expect( + await processPackageJob(jobs, job.id, (job) => + prepareQueuedPackage(packagePreparation(env), job) + ) + ).toBe("retry"); + expect(await jobs.get(job.id)).toMatchObject({ + status: "queued", + attempts: 1, + error: { code: "SOURCE_UNAVAILABLE", status: 503 } + }); + b2.objects.set(key, staged); + expect( + await processPackageJob(jobs, job.id, (job) => + prepareQueuedPackage(packagePreparation(env), job) + ) + ).toBe("ack"); + expect(await jobs.get(job.id)).toMatchObject({ + status: "ready", + attempts: 2, + error: null + }); + }); + it("recovers an expired processing lease without accepting stale completion", async () => { + const response = await upload( + manifest(), + undefined, + undefined, + "owner", + false + ); + const { result: job } = (await response.json()) as { + result: { id: string }; + }; + const jobs = new PackageJobsDatabase(db); + expect(await jobs.claim(job.id, "first")).not.toBeNull(); + expect(await jobs.claim(job.id, "second")).toBeNull(); + await db + .prepare( + "UPDATE package_jobs SET lease_until=datetime('now','-1 minute') WHERE id=?" + ) + .bind(job.id) + .run(); + const claimed = await jobs.claim(job.id, "second"); + expect(claimed?.attempts).toBe(2); + await jobs.fail( + job.id, + "first", + { code: "STALE", message: "Old worker", status: 503 }, + false + ); + expect((await jobs.get(job.id))!.status).toBe("processing"); + const artifactId = await prepareQueuedPackage( + packagePreparation(env), + claimed! + ); + await jobs.complete(job.id, "second", artifactId); + expect((await jobs.get(job.id))!.status).toBe("ready"); + }); + it("claims a job once across concurrent deliveries", async () => { + const response = await upload( + manifest(), + undefined, + undefined, + "owner", + false + ); + const { result: job } = (await response.json()) as { + result: { id: string }; + }; + const jobs = new PackageJobsDatabase(db); + const prepare = vi.fn( + ( + job: import("../../../../src/services/extensions/v2/packages/jobs").PackageJob + ) => prepareQueuedPackage(packagePreparation(env), job) + ); + await Promise.all([ + processPackageJob(jobs, job.id, prepare), + processPackageJob(jobs, job.id, prepare) + ]); + expect(prepare).toHaveBeenCalledTimes(1); + expect((await jobs.get(job.id))!.status).toBe("ready"); + }); +}); + +describe("stored directory packages", () => { + it("uses separate B2 buckets for ZIPs and icons through preparation, correction and approval", async () => { + const requests = b2.calls; + const writer = new ZipWriter(new Uint8ArrayWriter(), options); + await writer.add("Service.php", new TextReader("' + ) + ); + const first = await built(manifest(), await writer.close()); + const revised = await built( + { ...manifest(), description: "Rebuilt from B2 source" }, + undefined, + first.source_id + ); + expect(revised.source_id).toBe(first.source_id); + const uploads = requests.filter((request) => request.method === "PUT"); + expect(uploads).toHaveLength(4); // source, two manifests and one shared icon + expect( + uploads.filter((request) => + new URL(request.url).pathname.startsWith( + "/test-packages/local/packages/" + ) + ) + ).toHaveLength(3); + expect( + uploads.filter((request) => + new URL(request.url).pathname.startsWith("/test-assets/extensions/") + ) + ).toHaveLength(1); + const path = `${root}/packages/${revised.id}/download`; + const before = requests.length; + expect((await get(path, {})).status).toBe(404); + expect(requests.length).toBe(before); + const owner = await get(path, await authHeaders("owner")); + expect(owner.status).toBe(200); + expect(await sha256(new Uint8Array(await owner.arrayBuffer()))).toBe( + revised.sha256 + ); + expect((await get(`${root}/packages/${revised.id}/icon`, {})).status).toBe( + 404 + ); + expect( + ( + await get( + `${root}/packages/${revised.id}/icon`, + await authHeaders("owner") + ) + ).status + ).toBe(200); + const revision = await submitted(revised); + expect((await approve(revision)).status).toBe(200); + const publicDownload = await get(path, {}); + expect(publicDownload.status).toBe(200); + expect(publicDownload.headers.get("Cache-Control")).toBe( + "private, no-store" + ); + expect( + await sha256(new Uint8Array(await publicDownload.arrayBuffer())) + ).toBe(revised.sha256); + }); + + it("keeps uploads private until approval and publishes the exact reviewed ZIP", async () => { + const artifact = await built(); + const path = `${root}/packages/${artifact.id}/download`; + expect((await get(path, {})).status).toBe(404); + expect((await get(path, await authHeaders("stranger"))).status).toBe(404); + expect((await get(path, await authHeaders("owner"))).status).toBe(200); + const revision = await submitted(artifact); + expect((await get(path, {})).status).toBe(404); + expect((await approve(revision)).status).toBe(200); + const download = await get(path, {}); + expect(download.status).toBe(200); + expect(download.headers.get("Cache-Control")).toContain("no-store"); + const bytes = new Uint8Array(await download.arrayBuffer()); + expect(await sha256(bytes)).toBe(artifact.sha256); + const files = await unpack(bytes); + expect(files["Service.php"]).toBe(" { + const writer = new ZipWriter(new Uint8ArrayWriter(), options); + await writer.add("Service.php", new TextReader("' + ) + ); + const artifact = await built(manifest(), await writer.close()); + const path = `${root}/packages/${artifact.id}/icon`; + expect((await get(path, {})).status).toBe(404); + expect((await get(path, await authHeaders("stranger"))).status).toBe(404); + const preview = await get(path, await authHeaders("owner")); + expect(preview.status).toBe(200); + expect(preview.headers.get("Content-Type")).toBe("image/svg+xml"); + expect(preview.headers.get("Content-Security-Policy")).toContain("sandbox"); + expect(await preview.text()).toContain("(); + b2.objects.delete(packageKey(object!.object_id)); + // Reading the icon must not depend on opening the ZIP. + expect((await get(path, await authHeaders("owner"))).status).toBe(200); + expect((await approve(await submitted(artifact))).status).toBe(200); + expect((await get(path, {})).status).toBe(200); + const published = await getExtension(db, "package-demo"); + expect(published?.icon_url).toBe( + `https://extensions-assets.fossbilling.net/${String(before!.svg_object_id)}` + ); + await db + .prepare( + "UPDATE extensions SET delisted_at=CURRENT_TIMESTAMP WHERE identifier='package-demo'" + ) + .run(); + expect((await get(path, {})).status).toBe(404); + }); + it("sanitizes SVG scripts, events, foreign content, and stylesheets while preserving vector artwork", async () => { + const original = ` + +
HTML
 + + + + `; + const writer = new ZipWriter(new Uint8ArrayWriter(), options); + await writer.add("icon.svg", new TextReader(original)); + const artifact = await built(manifest(), await writer.close()); + const response = await get( + `${root}/packages/${artifact.id}/icon`, + await authHeaders("owner") + ); + const svg = await response.text(); + expect(response.headers.get("Content-Type")).toBe("image/svg+xml"); + expect(response.headers.get("Content-Security-Policy")).toBe( + "sandbox; default-src 'none'; style-src 'unsafe-inline'; img-src data:" + ); + expect(response.headers.get("X-Content-Type-Options")).toBe("nosniff"); + expect(svg).not.toMatch( + /script|onload|foreignObject|untrusted\.example|xml-stylesheet/ + ); + expect(svg).toContain("linearGradient"); + expect(svg).toContain("url(#paint)"); + expect(svg).toContain('viewBox="0 0 24 24"'); + const row = await db + .prepare( + "SELECT svg_object_id,avif_object_id,png_object_id FROM package_icons WHERE source_id=?" + ) + .bind(artifact.source_id) + .first<{ + svg_object_id: string; + avif_object_id: null; + png_object_id: null; + }>(); + expect(row?.svg_object_id).toMatch( + /^extensions\/[a-f0-9-]{36}\/icons\/[a-f0-9-]{36}\.svg$/ + ); + expect(row?.avif_object_id).toBeNull(); + expect(row?.png_object_id).toBeNull(); + const download = await get( + `${root}/packages/${artifact.id}/download`, + await authHeaders("owner") + ); + const zip = new ZipReader( + new Uint8ArrayReader(new Uint8Array(await download.arrayBuffer())), + options + ); + try { + const icon = (await zip.getEntries()).find( + (entry) => entry.filename === "icon.svg" + )!; + if (icon.directory) throw new Error("Expected icon file"); + expect(await icon.getData(new TextWriter())).toBe(original); + } finally { + await zip.close(); + } + }); + it.each([ + "', + '" + ])("rejects malformed, non-SVG, and oversized SVG files %#", async (svg) => { + const writer = new ZipWriter(new Uint8ArrayWriter(), options); + await writer.add("icon.svg", new TextReader(svg)); + expect((await upload(manifest(), await writer.close())).status).toBe(422); + expect( + await db.prepare("SELECT count(*) AS count FROM package_sources").first() + ).toEqual({ count: 0 }); + }); + it("sanitizes SVG content even with a raster filename", async () => { + const writer = new ZipWriter(new Uint8ArrayWriter(), options); + await writer.add( + "icon.png", + new TextReader( + '' + ) + ); + const artifact = await built(manifest(), await writer.close()); + const response = await get( + `${root}/packages/${artifact.id}/icon`, + await authHeaders("owner") + ); + expect(response.headers.get("Content-Type")).toBe("image/svg+xml"); + expect(await response.text()).not.toContain("onload"); + }); + it.each(["png", "webp", "avif"] as const)( + "converts icon.%s to bounded AVIF and PNG images", + async (format) => { + const svg = + ''; + const image = await env + .PACKAGE_IMAGES!.input(new Blob([svg]).stream()) + .output({ format: `image/${format}` }); + const original = new Uint8Array(await image.response().arrayBuffer()); + const writer = new ZipWriter(new Uint8ArrayWriter(), options); + await writer.add(`icon.${format}`, new Uint8ArrayReader(original)); + const artifact = await built(manifest(), await writer.close()); + const response = await get( + `${root}/packages/${artifact.id}/icon`, + await authHeaders("owner") + ); + const avif = await get(`${root}/packages/${artifact.id}/icon`, { + ...(await authHeaders("owner")), + Accept: "image/avif,image/png" + }); + expect(avif.headers.get("Content-Type")).toBe("image/avif"); + expect( + new TextDecoder().decode((await avif.arrayBuffer()).slice(4, 12)) + ).toBe("ftypavif"); + expect( + ( + await get(`${root}/packages/${artifact.id}/icon`, { + ...(await authHeaders("owner")), + Accept: "image/avif;q=0,image/png" + }) + ).headers.get("Content-Type") + ).toBe("image/png"); + const info = await env.PACKAGE_IMAGES!.info(response.body!); + expect(info).toMatchObject({ + format: "image/png", + width: 256, + height: 128 + }); + const download = await get( + `${root}/packages/${artifact.id}/download`, + await authHeaders("owner") + ); + const zip = new ZipReader( + new Uint8ArrayReader(new Uint8Array(await download.arrayBuffer())), + options + ); + try { + const entry = (await zip.getEntries()).find( + (entry) => entry.filename === `icon.${format}` + )!; + if (entry.directory) throw new Error("Expected icon file"); + expect(await entry.getData(new Uint8ArrayWriter())).toEqual(original); + } finally { + await zip.close(); + } + } + ); + + it("lets moderators rebuild published source for a correction without uploading new files", async () => { + const artifact = await built(); + const revision = await submitted(artifact); + expect((await approve(revision)).status).toBe(200); + const corrected = { ...manifest(), description: "Moderator correction" }; + const response = await upload( + corrected, + undefined, + artifact.source_id, + "moderator" + ); + expect(response.status).toBe(200); + const next = PackageArtifactSchema.parse( + ((await response.json()) as { result: unknown }).result + ); + expect(next.source_id).toBe(artifact.source_id); + expect(next.manifest.description).toBe(corrected.description); + expect( + (await upload(corrected, await zip(), undefined, "moderator")).status + ).toBe(403); + expect( + ( + await post( + `${root}/extensions/package-demo/moderator-correct`, + await authHeaders("moderator"), + { + ...content(next), + correction_note: "Correct the package metadata" + } + ) + ).status + ).toBe(200); + }); + + it("rejects invalid icons without reserving the source version", async () => { + const writer = new ZipWriter(new Uint8ArrayWriter(), options); + await writer.add("icon.png", new TextReader("not an image")); + const response = await upload(manifest(), await writer.close()); + expect(response.status).toBe(422); + expect(await response.json()).toMatchObject({ + error: { code: "INVALID_ICON" } + }); + expect( + await db.prepare("SELECT count(*) AS count FROM package_sources").first() + ).toEqual({ count: 0 }); + expect((await upload()).status).toBe(200); + }); + + async function iconSource(format: "svg" | "png" = "svg") { + const svg = + ''; + const bytes = + format === "svg" + ? new TextEncoder().encode(svg) + : new Uint8Array( + await ( + await env + .PACKAGE_IMAGES!.input(new Blob([svg]).stream()) + .output({ format: "image/png" }) + ) + .response() + .arrayBuffer() + ); + const writer = new ZipWriter(new Uint8ArrayWriter(), options); + await writer.add( + `icon.${format}`, + new Uint8ArrayReader(new Uint8Array(bytes)) + ); + return writer.close(); + } + it("stores pending icons at unlisted B2 URLs using the permanent draft UUID", async () => { + const artifact = await built(manifest(), await iconSource()); + const draft = (await getExtension(db, "package-demo"))!; + expect(draft.id).toMatch(UUID_V7); + expect(draft.published_at).toBeNull(); + expect( + (await get(`${root}/extensions/package-demo`, await authHeaders("owner"))) + .status + ).toBe(200); + expect((await get(`${root}/extensions/package-demo`, {})).status).toBe(404); + const uploads = b2.calls.filter( + (r) => + r.method === "PUT" && + new URL(r.url).pathname.startsWith("/test-assets/") + ); + expect(uploads).toHaveLength(1); + const upload = uploads[0]; + const objectKey = new URL(upload.url).pathname.replace("/test-assets/", ""); + expect(objectKey).toMatch( + new RegExp(`^extensions/${draft.id}/icons/[a-f0-9-]{36}\\.svg$`) + ); + const callsBefore = b2.calls.length; + expect((await get(`${root}/packages/${artifact.id}/icon`, {})).status).toBe( + 404 + ); + expect( + ( + await get( + `${root}/packages/${artifact.id}/icon`, + await authHeaders("stranger") + ) + ).status + ).toBe(404); + expect(b2.calls).toHaveLength(callsBefore); + const preview = await get( + `${root}/packages/${artifact.id}/icon`, + await authHeaders("owner") + ); + expect(preview.status).toBe(200); + expect(preview.headers.get("Cache-Control")).toBe("private, no-store"); + expect(await preview.text()).toContain(" + r.method === "PUT" && + new URL(r.url).pathname.startsWith("/test-assets/") + ) + ).toHaveLength(1); + }); + it("fails preparation on asset upload errors and retries without changing storage backends", async () => { + const source = await iconSource(); + const original = b2.fetch.getMockImplementation()!; + b2.fetch.mockImplementation(async (request) => + request.method === "PUT" && + new URL(request.url).pathname.startsWith("/test-assets/") + ? new Response(null, { status: 503 }) + : original(request) + ); + expect((await upload(manifest(), source)).status).toBe(503); + expect( + await db + .prepare("SELECT count(*) AS count FROM package_artifacts") + .first() + ).toEqual({ count: 0 }); + b2.fetch.mockImplementation(original); + const artifact = await built(manifest(), source); + expect( + ( + await get( + `${root}/packages/${artifact.id}/icon`, + await authHeaders("owner") + ) + ).status + ).toBe(200); + expect((await approve(await submitted(artifact))).status).toBe(200); + expect((await getExtension(db, "package-demo"))?.icon_url).toContain( + "https://extensions-assets.fossbilling.net/extensions/" + ); + }); + it("publishes an icon when a valid extension ID is supplied in uppercase", async () => { + const artifact = await built(manifest(), await iconSource()); + const revision = await submitted(artifact); + await insertUser(db, { id: "moderator", is_moderator: 1 }); + expect( + ( + await post( + `${root}/extensions/PACKAGE-DEMO/revisions/${revision}/approve`, + await authHeaders("moderator"), + {} + ) + ).status + ).toBe(200); + expect((await getExtension(db, "package-demo"))?.icon_url).toMatch( + /^https:\/\/extensions-assets\.fossbilling\.net\// + ); + }); + it("publishes AVIF and PNG with one immutable image identity for browser fallback", async () => { + const artifact = await built(manifest(), await iconSource("png")); + expect((await approve(await submitted(artifact))).status).toBe(200); + const uploads = b2.calls.filter( + (request) => + request.method === "PUT" && + new URL(request.url).pathname.startsWith("/test-assets/") + ); + expect(uploads).toHaveLength(2); + const extension = (await getExtension(db, "package-demo"))!; + expect(new URL(uploads[0].url).pathname).toMatch( + new RegExp( + `^/test-assets/extensions/${extension.id}/icons/[a-f0-9-]{36}\\.avif$` + ) + ); + expect(uploads[0].url.replace(/\.avif$/, ".png")).toBe(uploads[1].url); + expect( + uploads.map((request) => request.headers.get("Content-Type")) + ).toEqual(["image/avif", "image/png"]); + expect((await getExtension(db, "package-demo"))?.icon_url).toBe( + uploads[1].url.replace( + "https://s3.example.test/test-assets", + "https://extensions-assets.fossbilling.net" + ) + ); + }); + + it("serves a generic icon for an authorized package without an icon", async () => { + const artifact = await built(); + const response = await get( + `${root}/packages/${artifact.id}/icon`, + await authHeaders("owner") + ); + expect(response.status).toBe(200); + expect(await response.text()).toContain(" { + const source = await zip(); + const first = await built(manifest(), source); + const retry = await built(manifest(), source); + expect(retry.id).toBe(first.id); + const second = await built( + { ...manifest(), description: "Corrected metadata" }, + undefined, + first.source_id + ); + expect(second.source_id).toBe(first.source_id); + expect(second.manifest.version).toBe(first.manifest.version); + expect(second.id).not.toBe(first.id); + expect(second.sha256).not.toBe(first.sha256); + expect(second.download_url).not.toBe(first.download_url); + const rows = await db + .prepare("SELECT count(*) AS count FROM package_objects") + .first<{ count: number }>(); + expect(rows!.count).toBe(3); + }); + + it("reuses a ready artifact without reopening its stored source", async () => { + const artifact = await built(); + const source = await db + .prepare("SELECT object_id FROM package_sources WHERE id=?") + .bind(artifact.source_id) + .first<{ object_id: string }>(); + b2.objects.delete(packageKey(source!.object_id)); + expect((await built(manifest(), undefined, artifact.source_id)).id).toBe( + artifact.id + ); + expect( + ( + await upload( + { ...manifest(), description: "Needs a rebuild" }, + undefined, + artifact.source_id + ) + ).status + ).toBe(503); + }); + + it("requires a new version for changed source and hides foreign stored sources", async () => { + const first = await built(); + expect((await upload(manifest(), await zip("changed source"))).status).toBe( + 409 + ); + const changed = { ...manifest(), version: "1.1.0" }; + expect((await upload(changed, await zip("changed source"))).status).toBe( + 200 + ); + await seedDeveloper("other-developer", "other"); + expect( + (await upload(manifest(), undefined, first.source_id, "other")).status + ).toBe(403); // The package has already reserved the extension for its owner. + expect( + ( + await upload( + { ...manifest(), id: "other-extension" }, + undefined, + first.source_id, + "other" + ) + ).status + ).toBe(404); + }); + + it("rejects metadata/requirements/URL/checksum mismatches at submission", async () => { + const artifact = await built(); + for (const patch of [ + { description: "Not in ZIP" }, + { download_url: "https://evil.test/not-the-package.zip" }, + { requires: { fossbilling: ">=9.0.0" } }, + { + releases: [{ ...content(artifact).releases[0], sha256: "0".repeat(64) }] + }, + { + releases: [ + { + ...content(artifact).releases[0], + requires: { fossbilling: ">=9.0.0" } + } + ] + } + ]) { + const response = await post( + `${root}/extensions`, + await authHeaders("owner"), + { identifier: "package-demo", ...content(artifact), ...patch } + ); + expect(response.status).toBe(409); + } + expect( + await db + .prepare( + "SELECT count(*) AS count FROM extensions WHERE identifier='package-demo'" + ) + .first() + ).toEqual({ count: 1 }); + expect( + await db + .prepare("SELECT count(*) AS count FROM extension_revisions") + .first() + ).toEqual({ count: 0 }); + }); + + it("replaces a pending package, invalidates the old review, and preserves previously published bytes", async () => { + const first = await built(); + const revision = await submitted(first); + expect((await approve(revision)).status).toBe(200); + const second = await built( + { ...manifest(), description: "Correction" }, + undefined, + first.source_id + ); + const proposed = await put( + `${root}/extensions/package-demo`, + await authHeaders("owner"), + content(second) + ); + expect(proposed.status).toBe(202); + const oldPending = ( + (await proposed.json()) as { result: { revision_id: string } } + ).result.revision_id; + const third = await built( + { ...manifest(), description: "Final correction" }, + undefined, + first.source_id + ); + const replaced = await put( + `${root}/extensions/package-demo/revisions/${oldPending}`, + await authHeaders("owner"), + content(third) + ); + expect(replaced.status).toBe(202); + expect((await approve(oldPending)).status).toBe(404); + expect( + (await get(`${root}/packages/${second.id}/download`, {})).status + ).toBe(404); + expect( + ( + await approve( + ((await replaced.json()) as { result: { revision_id: string } }) + .result.revision_id + ) + ).status + ).toBe(200); + expect( + (await get(`${root}/packages/${first.id}/download`, {})).status + ).toBe(200); + expect( + (await get(`${root}/packages/${third.id}/download`, {})).status + ).toBe(200); + }); + + it("prevents reverting a managed listing to external downloads or mutating artifacts", async () => { + const artifact = await built(); + expect((await approve(await submitted(artifact))).status).toBe(200); + expect( + ( + await put( + `${root}/extensions/package-demo`, + await authHeaders("owner"), + sampleContent() + ) + ).status + ).toBe(409); + await expect( + db + .prepare("UPDATE package_artifacts SET sha256=? WHERE id=?") + .bind("0".repeat(64), artifact.id) + .run() + ).rejects.toThrow("immutable"); + await expect( + db + .prepare("UPDATE package_sources SET sha256=? WHERE id=?") + .bind("0".repeat(64), artifact.source_id) + .run() + ).rejects.toThrow("immutable"); + }); + + it("retains immutable package rows after job history is pruned", async () => { + const artifact = await built(); + expect((await approve(await submitted(artifact))).status).toBe(200); + await db.prepare("DELETE FROM package_jobs").run(); + for (const [table, column, id] of [ + ["package_artifacts", "id", artifact.id], + ["package_sources", "id", artifact.source_id], + ["package_icons", "source_id", artifact.source_id] + ]) + await expect( + db.prepare(`DELETE FROM ${table} WHERE ${column}=?`).bind(id).run() + ).rejects.toThrow("immutable"); + expect( + ( + await post( + `${root}/packages/cleanup`, + await authHeaders("moderator"), + {} + ) + ).status + ).toBe(200); + expect( + (await get(`${root}/packages/${artifact.id}/download`, {})).status + ).toBe(200); + }); + it("fetches artifact references together for a 100-release submission and rejects unknown artifacts", async () => { + const artifact = await built(); + const first = { ...content(artifact).releases[0], is_latest: true }; + const releases = [ + first, + ...Array.from({ length: 99 }, (_, i) => ({ + ...first, + is_latest: false, + tag: `${i + 2}.0.0`, + artifact_id: crypto.randomUUID() + })) + ]; + const headers = await authHeaders("owner"); + const prepared = vi.spyOn(db, "prepare"); + const response = await post(`${root}/extensions`, headers, { + identifier: "package-demo", + ...content(artifact), + releases + }); + const artifactQueries = prepared.mock.calls + .map(([sql]) => sql) + .filter((sql) => sql.includes("FROM package_artifacts")); + prepared.mockRestore(); + expect(response.status).toBe(409); + expect(await response.json()).toMatchObject({ + error: { message: "Release does not match its package artifact" } + }); + // Keep validation bounded rather than querying once per release. + expect(artifactQueries.length).toBeLessThanOrEqual(2); + expect( + ( + await post(`${root}/extensions`, headers, { + identifier: "package-demo", + ...content(artifact) + }) + ).status + ).toBe(201); + }); + it("revalidates the artifact at approval rather than trusting submission-time checks", async () => { + const artifact = await built(); + const revision = await submitted(artifact); + await db + .prepare( + "UPDATE extension_revisions SET content=json_set(content,'$.description','Changed after submission') WHERE id=?" + ) + .bind(revision) + .run(); + expect((await approve(revision)).status).toBe(409); + expect( + (await get(`${root}/packages/${artifact.id}/download`, {})).status + ).toBe(404); + }); + + it("detects corrupted stored source and package objects", async () => { + const artifact = await built(); + const source = await db + .prepare("SELECT object_id FROM package_sources WHERE id=?") + .bind(artifact.source_id) + .first<{ object_id: string }>(); + b2.objects.get(packageKey(source!.object_id))!.bytes = new Uint8Array( + new TextEncoder().encode("corrupt") + ).buffer; + expect( + ( + await upload( + { ...manifest(), description: "Correction" }, + undefined, + artifact.source_id + ) + ).status + ).toBe(503); + const stored = await db + .prepare("SELECT object_id FROM package_artifacts WHERE id=?") + .bind(artifact.id) + .first<{ object_id: string }>(); + b2.objects.get(packageKey(stored!.object_id))!.bytes = new Uint8Array( + new TextEncoder().encode("corrupt") + ).buffer; + expect( + ( + await get( + `${root}/packages/${artifact.id}/download`, + await authHeaders("owner") + ) + ).status + ).toBe(503); + }); + + it("concurrent identical uploads converge on one source and artifact", async () => { + const source = await zip(); + const [first, second] = await Promise.all([ + built(manifest(), source), + built(manifest(), source) + ]); + expect(first.id).toBe(second.id); + expect( + await db.prepare("SELECT count(*) AS count FROM package_sources").first() + ).toEqual({ count: 1 }); + expect( + await db + .prepare("SELECT count(*) AS count FROM package_artifacts") + .first() + ).toEqual({ count: 1 }); + expect( + await db.prepare("SELECT count(*) AS count FROM package_objects").first() + ).toEqual({ count: 3 }); // one duplicate staged upload remains eligible for cleanup + }); + + it("does not allow a removed managed release to return as an external download", async () => { + const first = await built(); + expect((await approve(await submitted(first))).status).toBe(200); + const second = await built({ ...manifest(), version: "1.1.0" }); + const nextContent = content(second); + const proposal = await put( + `${root}/extensions/package-demo`, + await authHeaders("owner"), + nextContent + ); + expect(proposal.status).toBe(202); + expect( + ( + await approve( + ((await proposal.json()) as { result: { revision_id: string } }) + .result.revision_id + ) + ).status + ).toBe(200); + const response = await put( + `${root}/extensions/package-demo`, + await authHeaders("owner"), + { + ...nextContent, + releases: [...nextContent.releases, sampleContent().releases[0]] + } + ); + expect(response.status).toBe(409); + }); + + it("publishes a manually selected older release and rejects competing latest flags", async () => { + const first = await built(); + const newer = await built({ ...manifest(), version: "2.0.0" }); + const payload = { + ...content(first), + releases: [ + { ...content(first).releases[0], is_latest: true }, + ...content(newer).releases + ] + }; + const conflict = await post( + `${root}/extensions`, + await authHeaders("owner"), + { + identifier: "package-demo", + ...payload, + releases: payload.releases.map((release) => ({ + ...release, + is_latest: true + })) + } + ); + expect(conflict.status).toBe(409); + const response = await post( + `${root}/extensions`, + await authHeaders("owner"), + { identifier: "package-demo", ...payload } + ); + expect(response.status).toBe(201); + expect( + ( + await approve( + ((await response.json()) as { result: { revision_id: string } }) + .result.revision_id + ) + ).status + ).toBe(200); + const version = await get("/extensions/v1/package-demo/version", {}); + expect(await version.text()).toBe("1.0.0"); + expect((await getExtension(db, "package-demo"))!.version).toBe("1.0.0"); + }); + + it("keeps production disabled until explicitly enabled with complete storage", async () => { + env.ENVIRONMENT = "production"; + expect((await upload()).status).toBe(503); // local mode cannot enable production + Object.assign(env, { EXTENSION_PACKAGES_MODE: "enabled" }); + expect((await upload()).status).toBe(503); // localhost URL / incomplete storage + env.EXTENSION_PACKAGES_BASE_URL = + "https://api.fossbilling.net/extensions/v2"; + expect((await upload()).status).toBe(503); // local object prefix cannot enable production + Object.assign(env, { EXTENSION_PACKAGES_S3_PREFIX: "production" }); + Reflect.deleteProperty(env, "EXTENSION_ASSETS_S3_SECRET_ACCESS_KEY"); + expect((await upload()).status).toBe(503); + Object.assign(env, { + EXTENSION_ASSETS_S3_SECRET_ACCESS_KEY: "test-secret" + }); + const requests = b2.calls; + const response = await upload( + manifest(), + undefined, + undefined, + "owner", + false + ); + expect(response.status).toBe(202); + expect(response.headers.get("Location")).toMatch( + /^https:\/\/api\.fossbilling\.net\/extensions\/v2\/package-jobs\// + ); + expect(requests.length).toBeGreaterThan(0); + expect( + requests.every((request) => request.url.includes("/production/packages/")) + ).toBe(true); + Object.assign(env, { EXTENSION_PACKAGES_S3_PREFIX: "local" }); + expect((await upload()).status).toBe(503); + Reflect.deleteProperty(env, "EXTENSION_PACKAGES_S3_PREFIX"); + Object.assign(env, { ENVIRONMENT: "local" }); + }); + + it("checks account ownership, upload bounds, invalid archives and disabled storage", async () => { + expect((await upload(manifest(), new Uint8Array([1, 2, 3]))).status).toBe( + 422 + ); + expect( + (await upload(manifest(), new Uint8Array(5 * 1024 * 1024 + 1))).status + ).toBe(413); + Reflect.deleteProperty(env, "EXTENSION_PACKAGES_S3_ENDPOINT"); + expect((await upload()).status).toBe(503); + Object.assign(env, { + EXTENSION_PACKAGES_S3_ENDPOINT: "https://s3.example.test" + }); + await db + .prepare("UPDATE users SET deleted_at=CURRENT_TIMESTAMP WHERE id='owner'") + .run(); + expect((await upload()).status).toBe(403); + env.EXTENSION_PACKAGES_MODE = "disabled"; + expect((await upload()).status).toBe(503); + }); + + it("leaves failed uploads reserved for cleanup and permits a safe retry", async () => { + b2.fetch.mockResolvedValueOnce(new Response(null, { status: 503 })); + expect((await upload()).status).toBe(503); + expect( + await db + .prepare("SELECT count(*) AS count FROM package_artifacts") + .first() + ).toEqual({ count: 0 }); + expect( + await db.prepare("SELECT count(*) AS count FROM package_objects").first() + ).toEqual({ count: 1 }); + expect((await upload()).status).toBe(200); + expect( + await db.prepare("SELECT count(*) AS count FROM package_sources").first() + ).toEqual({ count: 1 }); + }); + + it("counts abandoned reservations against quota and safely cleans them after a day", async () => { + const artifact = await built(); + const abandoned = crypto.randomUUID(); + await db + .prepare( + "INSERT INTO package_objects (id,created_by,bytes,created_at) VALUES (?,'owner',?,datetime('now','-2 days'))" + ) + .bind(abandoned, 512 * 1024 * 1024) + .run(); + const bytes = new Uint8Array(new TextEncoder().encode("failed upload")); + b2.objects.set(packageKey(abandoned), { + bytes: bytes.buffer, + checksum: await sha256(bytes), + version: "abandoned-version" + }); + expect( + ( + await upload( + { ...manifest(), description: "quota test" }, + undefined, + artifact.source_id + ) + ).status + ).toBe(409); + await insertUser(db, { id: "moderator", is_moderator: 1 }); + const cleanup = await post( + `${root}/packages/cleanup`, + await authHeaders("moderator") + ); + expect(cleanup.status).toBe(200); + expect(await cleanup.json()).toEqual({ result: { deleted: 1 } }); + expect(b2.objects.has(packageKey(abandoned))).toBe(false); + expect( + b2.calls.find((request) => request.method === "DELETE")?.url + ).toContain("versionId=abandoned-version"); + expect( + ( + await upload( + { ...manifest(), description: "quota test" }, + undefined, + artifact.source_id + ) + ).status + ).toBe(200); + }); +}); diff --git a/vitest.config.ts b/vitest.config.ts index 5555db35..0fd362d1 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -18,6 +18,7 @@ export default defineConfig({ cloudflareTest({ wrangler: { configPath: "./wrangler.jsonc" }, miniflare: { + images: { binding: "PACKAGE_IMAGES" }, // Add test environment variables bindings: { GITHUB_TOKEN: "test-github-token", @@ -68,7 +69,8 @@ export default defineConfig({ exclude: [ "**/node_modules/**", "**/test/lib/adapters/node/**", - "**/test/services/extensions/v2/migrations.test.ts" + "**/test/services/extensions/v2/migrations.test.ts", + "**/*.node.test.ts" ], // Test timeout configuration diff --git a/vitest.node.config.ts b/vitest.node.config.ts index d127eb85..11e93c0b 100644 --- a/vitest.node.config.ts +++ b/vitest.node.config.ts @@ -5,7 +5,8 @@ export default defineConfig({ environment: "node", include: [ "test/lib/adapters/node/**/*.test.ts", - "test/services/extensions/v2/migrations.test.ts" + "test/services/extensions/v2/migrations.test.ts", + "test/services/extensions/v2/*.node.test.ts" ], testTimeout: 10000, diff --git a/worker-configuration.d.ts b/worker-configuration.d.ts index 45537ba4..5e25d541 100644 --- a/worker-configuration.d.ts +++ b/worker-configuration.d.ts @@ -1,26 +1,29 @@ /* eslint-disable */ -// Generated by Wrangler by running `wrangler types --env-interface=CloudflareBindings` (hash: e27e965b107adae819c46ccc0379af8f) -// Runtime types generated with workerd@1.20261006.1 2026-10-01 +// Generated by Wrangler by running `wrangler types --env-interface=CloudflareBindings` (hash: e610663ea57a18e0636f4e14dece74ae) +// Runtime types generated with workerd@1.20261104.1 2026-10-01 interface __BaseEnv_CloudflareBindings { AUTH_KV: KVNamespace; CACHE_KV: KVNamespace; DOWNLOAD_BUCKET: R2Bucket; DB_CENTRAL_ALERTS: D1Database; DB_EXTENSIONS: D1Database; + EXTENSION_PACKAGE_QUEUE: Queue; EXTENSION_WRITE_RATE_LIMITER: RateLimit; PROFILE_CREATION_RATE_LIMITER: RateLimit; + PACKAGE_IMAGES: ImagesBinding; ENVIRONMENT: "production"; + EXTENSION_PACKAGES_MODE: "disabled"; + EXTENSION_PACKAGES_BASE_URL: "https://api.fossbilling.net/extensions/v2"; + EXTENSION_ASSETS_PUBLIC_URL: "https://extensions-assets.fossbilling.net"; EXTENSIONS_RETENTION_MODE: "dry-run"; EXTENSIONS_V2_EMAIL_PROVIDER: "mxroute"; - EXTENSIONS_V2_MXROUTE_SERVER: "wednesday.mxrouting.net"; - EXTENSIONS_V2_MXROUTE_USERNAME: "extensions@fossbilling.org"; + M2M_ISSUER: "https://auth.fossbilling.net"; + M2M_JWKS_URL: "https://auth.fossbilling.net/.well-known/jwks.json"; + M2M_AUDIENCE: "https://api.fossbilling.net"; GITHUB_TOKEN: string; ASSERTION_SIGNING_SECRET: string; EXTENSIONS_V2_MXROUTE_PASSWORD: string; EXTENSIONS_REVALIDATE_SECRET: string; - M2M_ISSUER: "https://auth.fossbilling.net"; - M2M_JWKS_URL: "https://auth.fossbilling.net/.well-known/jwks.json"; - M2M_AUDIENCE: "https://api.fossbilling.net"; PREVIEW_GITHUB_BUDGET: DurableObjectNamespace; EXTENSIONS_FRONTEND: Fetcher /* extensions */; } @@ -36,7 +39,7 @@ type StringifyValues> = { [Binding in keyof EnvType]: EnvType[Binding] extends string ? EnvType[Binding] : string; }; declare namespace NodeJS { - interface ProcessEnv extends StringifyValues> {} + interface ProcessEnv extends StringifyValues> {} } declare module "*.txt" { const value: string; @@ -488,6 +491,7 @@ interface ExecutionContext { readonly props: Props; cache?: CacheContext; readonly access?: CloudflareAccessContext; + mapVirtualHost(fetcher: Fetcher, port: number): string; tracing: Tracing; abort(reason?: any): void; } @@ -588,9 +592,14 @@ type DurableObjectRoutingMode = "primary-only"; interface DurableObjectNamespaceGetDurableObjectOptions { locationHint?: DurableObjectLocationHint; routingMode?: DurableObjectRoutingMode; + retry?: DurableObjectRetryOptions; } interface DurableObjectClass<_T extends Rpc.DurableObjectBranded | undefined = undefined> { } +interface DurableObjectRetryOptions { + maxRetries?: number; + timeoutMs?: number; +} interface DurableObjectState { waitUntil(promise: Promise): void; readonly exports: Cloudflare.Exports; @@ -3346,13 +3355,7 @@ declare abstract class SqlStorageStatement { } type SqlStorageValue = ArrayBuffer | string | number | null; declare abstract class SqlStorageCursor> { - next(): { - done?: false; - value: T; - } | { - done: true; - value?: never; - }; + next(): IteratorResult; toArray(): T[]; one(): T; raw(): IterableIterator; @@ -3511,9 +3514,9 @@ interface ContainerSnapshotOptions { } type ContainerStartupOptions = { entrypoint?: string[]; - enableInternet: boolean; + enableInternet?: boolean; env?: Record; - instance?: "lite" | "standard-1" | "standard-2" | "standard-3" | "standard-4" | ContainerStartResources; + instance?: "lite" | "basic" | "standard-1" | "standard-2" | "standard-3" | "standard-4" | ContainerStartResources; labels?: Record; directorySnapshots?: ContainerDirectorySnapshotRestoreParams[]; } & ({ @@ -3678,6 +3681,7 @@ interface Tracing { } declare abstract class Span { get isTraced(): boolean; + spanContext(): TracingSpanContext; setAttribute(key: string, value: boolean | number | string): this; setAttributes(attributes: Record): this; recordException(exception: string | { @@ -3705,6 +3709,11 @@ interface TracingSpanStatus { code: TracingSpanStatusCode; message?: string; } +interface TracingSpanContext { + traceId: string; + spanId: string; + traceFlags: number; +} /** * Represents the identity of a user authenticated via Cloudflare Access. * This matches the result of calling /cdn-cgi/access/get-identity. @@ -11583,20 +11592,16 @@ interface BrowserRunBaseOptions { */ cacheTTL?: number; } -/** - * Backend selection, mixed into the options of the quick actions that support it. - * Deliberately not part of `BrowserRunBaseOptions`: `scrape`, `links` and `snapshot` - * reject an alternate backend, so they must not accept the field. - */ +/** Backend selection for quick actions. */ interface BrowserRunAlternateBackendOptions { /** Render with an alternate browser backend instead of the default one. */ browser?: 'kitesurf'; } /** Common options shared by all quick actions. Exactly one of `url` or `html` must be provided.*/ -type BrowserRunCommonOptions = (BrowserRunBaseOptions & { +type BrowserRunCommonOptions = BrowserRunBaseOptions & BrowserRunAlternateBackendOptions & ({ /** URL to navigate to, e.g. `"https://example.com"`. */ url: string; -}) | (BrowserRunBaseOptions & { +} | { /** Set the HTML content of the page directly. */ html: string; }); @@ -11626,7 +11631,7 @@ type BrowserRunScreenshotOptions = BrowserRunCommonOptions & { scrollPage?: boolean; /** @see https://pptr.dev/api/puppeteer.screenshotoptions */ screenshotOptions?: BrowserRunPuppeteerScreenshotOptions; -} & BrowserRunAlternateBackendOptions; +}; type BrowserRunPDFOptions = BrowserRunCommonOptions & { /** @see https://pptr.dev/api/puppeteer.pdfoptions */ pdfOptions?: { @@ -11662,7 +11667,7 @@ type BrowserRunPDFOptions = BrowserRunCommonOptions & { /** @default 30000 */ timeout?: number; }; -} & BrowserRunAlternateBackendOptions; +}; type BrowserRunScrapeOptions = BrowserRunCommonOptions & { /** CSS selectors to scrape. At least one element is required. */ elements: Array<{ @@ -11697,7 +11702,7 @@ type BrowserRunAccessibilityTreeOptions = BrowserRunCommonOptions & { * HTTP 200; a malformed selector is an error. */ root?: string; -} & BrowserRunAlternateBackendOptions; +}; interface BrowserRunJsonBaseOptions { /** Custom AI models to try in order. Max 3. Falls back to next on error. */ custom_ai?: Array<{ @@ -11711,7 +11716,7 @@ interface BrowserRunJsonBaseOptions { * Options for the `json` quick action. * At least one of `prompt` or `response_format` must be provided. */ -type BrowserRunJsonOptions = BrowserRunCommonOptions & BrowserRunAlternateBackendOptions & BrowserRunJsonBaseOptions & ({ +type BrowserRunJsonOptions = BrowserRunCommonOptions & BrowserRunJsonBaseOptions & ({ /** Natural-language prompt describing what data to extract. */ prompt: string; /** Structured output schema for the AI model. @see https://developers.cloudflare.com/workers-ai/json-mode/ */ @@ -11722,8 +11727,8 @@ type BrowserRunJsonOptions = BrowserRunCommonOptions & BrowserRunAlternateBacken /** Structured output schema for the AI model. @see https://developers.cloudflare.com/workers-ai/json-mode/ */ response_format: AiTextGenerationResponseFormat; }); -type BrowserRunContentOptions = BrowserRunCommonOptions & BrowserRunAlternateBackendOptions; -type BrowserRunMarkdownOptions = BrowserRunCommonOptions & BrowserRunAlternateBackendOptions; +type BrowserRunContentOptions = BrowserRunCommonOptions; +type BrowserRunMarkdownOptions = BrowserRunCommonOptions; type BrowserRunRedirectHop = { /** URL that returned the redirect. */ url: string; @@ -13261,20 +13266,30 @@ interface Disposable { } declare module 'cloudflare:durable-objects' { /** - * Marks a Durable Object method as safe to run more than once. The runtime may then retry calls - * to it after a disconnect that might have happened after the method started. It does not + * Marks Durable Object methods as safe to run more than once. The runtime may then retry calls + * to them after a disconnect that might have happened after the method started. It does not * enable retries or change how many are made. * + * Mark methods on the class prototype, for example from a static block: + * + * ```ts + * class Counter extends DurableObject { + * static { + * retryable(this.prototype.reset); + * } + * async reset() {} + * } + * ``` + * + * The mark is on the function itself. A subclass inherits it, but an override, a bound copy, or + * a wrapper is not marked unless it is also passed to `retryable()`. + * * Each attempt creates its own return value, so if the method returns an `RpcTarget`, its * `[Symbol.dispose]()` may run once per attempt. * - * Only public instance methods can be decorated. When composing decorators, apply `@retryable` - * outermost (first in source order) so it marks the method that is finally installed. - * Otherwise `@retryable` has no effect, and constructing the object logs a warning. - * Requires standard (not `experimentalDecorators`) decorators and a bundler that transforms - * them, such as Wrangler. + * @throws TypeError if an argument is not a function. */ - export function retryable(value: (this: This, ...args: Args) => Return, context: ClassMethodDecoratorContext Return>): (this: This, ...args: Args) => Return; + export function retryable(...methods: ((...args: never[]) => unknown)[]): void; } /** * The returned data after sending an email @@ -13416,7 +13431,10 @@ declare module "cloudflare:email" { * Evaluation context for targeting rules. * Keys are attribute names (e.g. "userId", "country"), values are the attribute values. */ -type FlagshipEvaluationContext = Record; +type FlagshipEvaluationContextValue = string | number | boolean | null | FlagshipEvaluationContextValue[] | { + [key: string]: FlagshipEvaluationContextValue; +}; +type FlagshipEvaluationContext = Record; interface FlagshipEvaluationDetails { flagKey: string; value: T; @@ -13425,6 +13443,7 @@ interface FlagshipEvaluationDetails { errorCode?: string | undefined; errorMessage?: string | undefined; } +type FlagshipWidenedValue = T extends boolean ? boolean : T extends string ? string : T extends number ? number : T; interface FlagshipEvaluationError extends Error { } /** @@ -13452,8 +13471,23 @@ declare abstract class Flagship { * @param flagKey The key of the flag to evaluate. * @param defaultValue Optional default value returned when evaluation fails. * @param context Optional evaluation context for targeting rules. + * @deprecated Use getValue() instead. */ get(flagKey: string, defaultValue?: unknown, context?: FlagshipEvaluationContext): Promise; + /** + * Get a flag value, inferring its expected type from the default value. + * @param flagKey The key of the flag to evaluate. + * @param defaultValue Default value returned when evaluation fails or the flag type does not match. + * @param context Optional evaluation context for targeting rules. + */ + getValue | unknown[]>(flagKey: string, defaultValue: T, context?: FlagshipEvaluationContext): Promise>; + /** + * Get a flag value with full evaluation details, inferring its expected type from the default value. + * @param flagKey The key of the flag to evaluate. + * @param defaultValue Default value returned when evaluation fails or the flag type does not match. + * @param context Optional evaluation context for targeting rules. + */ + getDetails | unknown[]>(flagKey: string, defaultValue: T, context?: FlagshipEvaluationContext): Promise>>; /** * Get a boolean flag value. * @param flagKey The key of the flag to evaluate. @@ -14254,7 +14288,7 @@ declare namespace Rpc { // Structured cloneables BaseType // Structured cloneable composites - | Map ? Serializable : never, T extends Map ? Serializable : never> | Set ? Serializable : never> | ReadonlyArray ? Serializable : never> | { + | Map ? Serializable : never, T extends Map ? Serializable : never> | Set ? Serializable : never> | ReadonlyMap ? Serializable : never, T extends ReadonlyMap ? Serializable : never> | ReadonlySet ? Serializable : never> | ReadonlyArray ? Serializable : never> | { [K in keyof T]: K extends number | string ? Serializable : never; } // Special types @@ -14272,7 +14306,7 @@ declare namespace Rpc { type BaseType = void | undefined | null | boolean | number | bigint | string | TypedArray | ArrayBuffer | DataView | Date | Error | RegExp | ReadableStream | WritableStream | Request | Response | Headers; // Recursively rewrite all `Stubable` types with `Stub`s // prettier-ignore - type Stubify = T extends Stubable ? Stub : T extends Map ? Map, Stubify> : T extends Set ? Set> : T extends Array ? Array> : T extends ReadonlyArray ? ReadonlyArray> : T extends BaseType ? T : T extends { + type Stubify = T extends Stubable ? Stub : T extends Map ? Map, Stubify> : T extends Set ? Set> : T extends ReadonlyMap ? ReadonlyMap, Stubify> : T extends ReadonlySet ? ReadonlySet> : T extends Array ? Array> : T extends ReadonlyArray ? ReadonlyArray> : T extends BaseType ? T : T extends { [key: string | number]: any; } ? { [K in keyof T]: Stubify; @@ -14281,7 +14315,7 @@ declare namespace Rpc { // Note we use `StubBase` instead of `Stub` here to avoid circular dependencies: // `Stub` depends on `Provider`, which depends on `Unstubify`, which would depend on `Stub`. // prettier-ignore - type Unstubify = T extends StubBase ? V : T extends Map ? Map, Unstubify> : T extends Set ? Set> : T extends Array ? Array> : T extends ReadonlyArray ? ReadonlyArray> : T extends BaseType ? T : T extends { + type Unstubify = T extends StubBase ? V : T extends Map ? Map, Unstubify> : T extends Set ? Set> : T extends ReadonlyMap ? ReadonlyMap, Unstubify> : T extends ReadonlySet ? ReadonlySet> : T extends Array ? Array> : T extends ReadonlyArray ? ReadonlyArray> : T extends BaseType ? T : T extends { [key: string | number]: unknown; } ? { [K in keyof T]: Unstubify; @@ -15401,7 +15435,7 @@ declare namespace TailStream { } type EventOutcome = "ok" | "canceled" | "exception" | "unknown" | "killSwitch" | "daemonDown" | "exceededCpu" | "exceededMemory" | "loadShed" | "responseStreamDisconnected" | "scriptNotFound" | "internalError" | "exceededWallTime" | "aborted"; interface ScriptVersion { - readonly id: string; + readonly id?: string; readonly tag?: string; readonly message?: string; } @@ -15412,7 +15446,7 @@ declare namespace TailStream { } interface Onset { readonly type: "onset"; - readonly attributes: Attribute[]; + readonly attributes?: Attribute[]; // id for the span being opened by this Onset event. readonly spanId: string; readonly dispatchNamespace?: string; @@ -15551,6 +15585,7 @@ declare namespace TailStream { diagnosticChannel?: TailEventHandler; exception?: TailEventHandler; log?: TailEventHandler; + streamDiagEvent?: TailEventHandler; return?: TailEventHandler; attributes?: TailEventHandler; }; diff --git a/wrangler.jsonc b/wrangler.jsonc index ac30eea0..9ea8ef5c 100644 --- a/wrangler.jsonc +++ b/wrangler.jsonc @@ -20,13 +20,16 @@ "mode": "smart" }, "preview_urls": true, + // Preserve operational runtime variables managed in the Worker dashboard. "keep_vars": true, "vars": { "ENVIRONMENT": "production", + // Enable only after the queue, Images binding and B2 secrets are configured. + "EXTENSION_PACKAGES_MODE": "disabled", + "EXTENSION_PACKAGES_BASE_URL": "https://api.fossbilling.net/extensions/v2", + "EXTENSION_ASSETS_PUBLIC_URL": "https://extensions-assets.fossbilling.net", "EXTENSIONS_RETENTION_MODE": "dry-run", "EXTENSIONS_V2_EMAIL_PROVIDER": "mxroute", - "EXTENSIONS_V2_MXROUTE_SERVER": "wednesday.mxrouting.net", - "EXTENSIONS_V2_MXROUTE_USERNAME": "extensions@fossbilling.org", "M2M_ISSUER": "https://auth.fossbilling.net", "M2M_JWKS_URL": "https://auth.fossbilling.net/.well-known/jwks.json", "M2M_AUDIENCE": "https://api.fossbilling.net" @@ -60,6 +63,21 @@ "id": "0771957093ae481b9cd974ffa83f8263" } ], + "queues": { + "producers": [ + { "binding": "EXTENSION_PACKAGE_QUEUE", "queue": "extension-packages" } + ], + "consumers": [ + { + "queue": "extension-packages", + "max_batch_size": 1, + "max_batch_timeout": 1, + "max_retries": 5, + "max_concurrency": 1 + } + ] + }, + "images": { "binding": "PACKAGE_IMAGES", "remote": false }, "r2_buckets": [ { "binding": "DOWNLOAD_BUCKET",