From f0b0512d3cfb65f9f939d4cba7d9e49c45083c12 Mon Sep 17 00:00:00 2001 From: Adam Daley Date: Wed, 4 Nov 2026 02:52:44 +0000 Subject: [PATCH 1/5] Align extensions v2 with the manifest contract --- AGENTS.md | 4 +- README.md | 6 +- package-lock.json | 7 - package.json | 1 - src/services/extensions/v1/database.ts | 65 +- src/services/extensions/v1/index.ts | 2 +- src/services/extensions/v1/interfaces.ts | 15 +- src/services/extensions/v1/list-cache.ts | 2 +- src/services/extensions/v2/README.md | 32 +- src/services/extensions/v2/db/extensions.ts | 136 +- .../v2/db/migrations/0029_align_manifest.sql | 245 +++ .../v2/db/migrations/meta/0029_snapshot.json | 1369 +++++++++++++++++ .../v2/db/migrations/meta/_journal.json | 7 + src/services/extensions/v2/db/revisions.ts | 10 +- src/services/extensions/v2/db/schema.ts | 17 +- src/services/extensions/v2/revalidate.ts | 10 +- .../v2/routes/developer-profiles.ts | 8 +- .../extensions/v2/routes/moderation.ts | 14 +- src/services/extensions/v2/routes/notify.ts | 10 +- .../extensions/v2/routes/owner-extensions.ts | 6 +- .../extensions/v2/routes/public-extensions.ts | 4 +- .../extensions/v2/schemas/extensions.ts | 128 +- .../extensions/v2/schemas/manifest.ts | 73 + test/services/extensions/v1/index.test.ts | 6 +- test/services/extensions/v2/db-fixtures.ts | 2 +- .../extensions/v2/extension-writes.test.ts | 32 +- test/services/extensions/v2/harness.ts | 11 +- test/services/extensions/v2/manifest.test.ts | 148 ++ .../services/extensions/v2/migrations.test.ts | 127 +- .../v2/moderation-revalidate.test.ts | 10 +- .../services/extensions/v2/moderation.test.ts | 12 +- .../extensions/v2/public-extensions.test.ts | 28 +- .../extensions/v2/resource-bounds.test.ts | 25 +- wrangler.jsonc | 2 +- 34 files changed, 2328 insertions(+), 246 deletions(-) create mode 100644 src/services/extensions/v2/db/migrations/0029_align_manifest.sql create mode 100644 src/services/extensions/v2/db/migrations/meta/0029_snapshot.json create mode 100644 src/services/extensions/v2/schemas/manifest.ts create mode 100644 test/services/extensions/v2/manifest.test.ts diff --git a/AGENTS.md b/AGENTS.md index 18791ecd..012befdf 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -65,10 +65,10 @@ Each service documents its own contract and operational detail in its own `READM - Bindings, cron, and vars are defined in `wrangler.jsonc` (D1, KV, R2, Durable Objects, rate limiters, service bindings); keep names aligned with `CloudflareBindings`. - Use Wrangler secrets for production tokens instead of committing them. - `ASSERTION_SIGNING_SECRET`: HMAC key the extensions v2 API uses to verify short-lived bearer assertions minted by the extensions site (`src/lib/auth/bearer-assertion.ts`). Not sent over the wire — only signs/verifies server-side in each Worker. Add `ASSERTION_SIGNING_SECRET="..."` to `.dev.vars` for local dev; set via `wrangler secret put ASSERTION_SIGNING_SECRET` in production, matching the value configured in the extensions site's Worker. -- `EXTENSIONS_REVALIDATE_SECRET`: bearer token the extensions v2 API sends to the extensions site's `POST /api/revalidate` (over the `EXTENSIONS_FRONTEND` service binding) to purge the site's CDN-cached catalogue pages after content mutations. Must match the value configured in the extensions site's Worker. Same local/prod setup as `ASSERTION_SIGNING_SECRET`. +- `EXTENSIONS_REVALIDATE_SECRET`: bearer token the extensions v2 API sends to the extensions site's `POST /api/revalidate` (over the `EXTENSIONS_FRONTEND` service binding) to purge the site's CDN-cached extension pages after content mutations. Must match the value configured in the extensions site's Worker. Same local/prod setup as `ASSERTION_SIGNING_SECRET`. ## Cache Revalidation -Any endpoint that mutates catalogue-visible content (revision approve/reject, moderator-correct, delist/relist, developer approve, developer profile upsert via `PUT /developers/me`, profile deletion via `DELETE /developers/me`, claim approve/reject, extension withdraw) must call `revalidateCatalogue(c)` from `src/services/extensions/v2/revalidate.ts` after a successful write. +Any endpoint that mutates publicly visible content (revision approve/reject, moderator-correct, delist/relist, developer approve, developer profile upsert via `PUT /developers/me`, profile deletion via `DELETE /developers/me`, claim approve/reject, extension withdraw) must call `revalidateExtensions(c)` from `src/services/extensions/v2/revalidate.ts` after a successful write. Skipping it does not break correctness — the site's `maxAge`/`stale-while-revalidate` windows bound staleness — but changes then take minutes instead of seconds to appear. Profile deletion keeps its purge even though the `developers` tag is currently a no-op. diff --git a/README.md b/README.md index 19533057..e46bcd39 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # FOSSBilling API Worker -This is the API service that acts as the central hub for FOSSBilling instances. It serves version checks, system-wide alerts, the extensions catalogue, preview builds, and release stats. +This is the API service that acts as the central hub for FOSSBilling instances. It serves version checks, system-wide alerts, the extensions directory, preview builds, and release stats. Everything is built on [Hono](https://hono.dev), making it lightweight and fast. While we currently deploy this to Cloudflare Workers, the code is designed to be platform-agnostic. @@ -71,7 +71,7 @@ We use [Cloudflare D1](https://developers.cloudflare.com/d1/) and [KV](https://d - **R2 Bucket** (`DOWNLOAD_BUCKET`): Shared by versions, previews, and stats for release/preview zips — see [`src/services/previews/v1/README.md`](src/services/previews/v1/README.md) and the comment in `wrangler.jsonc` for which bucket this points at and why. - **Durable Object** (`PREVIEW_GITHUB_BUDGET`): Token bucket limiting previews GitHub calls. Policy lives in `src/services/previews/v1/budget.ts`. - **Rate limiters** (`EXTENSION_WRITE_RATE_LIMITER`, `PROFILE_CREATION_RATE_LIMITER`): Edge pacing for extension writes and profile creation. Durable quotas are enforced in D1; see [extensions v2 README](src/services/extensions/v2/README.md#limits-and-budgets). -- **Service binding** (`EXTENSIONS_FRONTEND`): Private binding to the Extensions site worker, used to purge cached catalogue pages after mutations. +- **Service binding** (`EXTENSIONS_FRONTEND`): Private binding to the Extensions site worker, used to purge cached extension pages after mutations. - **Cron** (`0 * * * *`): Hourly extension resource maintenance. See [extensions v2 README](src/services/extensions/v2/README.md#revisions-and-retention). ### Environment Variables @@ -79,7 +79,7 @@ We use [Cloudflare D1](https://developers.cloudflare.com/d1/) and [KV](https://d - `GITHUB_TOKEN`: A GitHub Personal Access Token (classic) with public repo read access. - `ASSERTION_SIGNING_SECRET`: Shared HMAC secret used to verify the short-lived bearer assertions minted by the Extensions site. Configure the same value in both Workers; it is never sent to clients. - `ASSERTION_SIGNING_SECRET_PREVIOUS`: Optional previous HMAC secret accepted during a signing-key rotation. -- `EXTENSIONS_REVALIDATE_SECRET`: Bearer token the API sends to the Extensions site's `POST /api/revalidate` (over the `EXTENSIONS_FRONTEND` service binding) to purge the site's CDN-cached catalogue pages after content mutations. Configure the same value in both Workers; it is never sent to clients. +- `EXTENSIONS_REVALIDATE_SECRET`: Bearer token the API sends to the Extensions site's `POST /api/revalidate` (over the `EXTENSIONS_FRONTEND` service binding) to purge the site's CDN-cached extension pages after content mutations. Configure the same value in both Workers; it is never sent to clients. - `EXTENSIONS_V2_EMAIL_PROVIDER`: `mxroute` or `resend` to send moderator notification emails, or unset/`disabled` to skip sending. - `EXTENSIONS_V2_EMAIL_FROM`: Sender address for notification emails (default `noreply@fossbilling.org`). - `EXTENSIONS_V2_EMAIL_REPLY_TO`: Optional Reply-To address for a monitored inbox. Omit when unset (no Reply-To header is sent) — notification emails do not invite replies. diff --git a/package-lock.json b/package-lock.json index 395778e9..bcfca4f8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,7 +16,6 @@ "drizzle-orm": "0.45.4", "hono": "4.13.14", "semver": "7.8.5", - "spdx-license-ids": "3.0.24", "zod": "4.6.7" }, "devDependencies": { @@ -5913,12 +5912,6 @@ "source-map": "^0.6.0" } }, - "node_modules/spdx-license-ids": { - "version": "3.0.24", - "resolved": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.24.tgz", - "integrity": "sha512-cLS9TtWkIQFyLkJ3/5aFQAOHOSKTlOs/7WDut/XPSdjom1fJhUdkepeJAKXa9Y05+FabHd0sti+Et559vDtkpQ==", - "license": "CC0-1.0" - }, "node_modules/std-env": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/std-env/-/std-env-4.3.0.tgz", diff --git a/package.json b/package.json index f9a4b5ad..2634ff95 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,6 @@ "drizzle-orm": "0.45.4", "hono": "4.13.14", "semver": "7.8.5", - "spdx-license-ids": "3.0.24", "zod": "4.6.7" }, "devDependencies": { diff --git a/src/services/extensions/v1/database.ts b/src/services/extensions/v1/database.ts index bdf9de76..e37cb434 100644 --- a/src/services/extensions/v1/database.ts +++ b/src/services/extensions/v1/database.ts @@ -1,4 +1,5 @@ -import { and, eq, isNotNull, isNull, sql } from "drizzle-orm"; +import { minVersion } from "semver"; +import { and, eq, isNotNull, isNull, or, sql } from "drizzle-orm"; import { ExtensionsDb } from "../../../lib/db"; import { extensions, developers } from "../v2/db/schema"; import { DatabaseResult } from "../../../lib/interfaces"; @@ -34,10 +35,10 @@ const EXTENSION_COLUMNS = { // non-null here because both queries below filter on published_at IS NOT NULL // and extensions_published_content_check makes that filter sufficient - a // published row cannot be missing any of them. That filter is also what keeps -// unreviewed extensions out of the v1 catalogue. delisted_at IS NULL is the -// other half: v1 shares this table with v2's public catalogue and must stay +// unreviewed extensions out of the v1 extension listing. delisted_at IS NULL is the +// other half: v1 shares this table with v2's public extension listing and must stay // in sync with what v2 hides, or a moderator delisting an extension would -// pull it from the v2 catalogue while it stayed visible here. +// pull it from the v2 extension listing while it stayed visible here. interface ExtensionRow { id: string; type: string; @@ -101,10 +102,22 @@ export class ExtensionsDatabase { .select(EXTENSION_COLUMNS) .from(extensions) .innerJoin(developers, eq(extensions.developerId, developers.id)) - .where(type ? and(published, eq(extensions.type, type)) : published); + .where( + type + ? and( + published, + type === "mod" + ? or( + eq(extensions.type, "mod"), + eq(extensions.type, "module") + ) + : eq(extensions.type, type) + ) + : published + ); // Offset pagination needs a deterministic order (the unpaginated // default deliberately stays unordered) - (LOWER(id), id) matches - // the published-catalogue index and is unique. + // the published-extension index and is unique. rows = page ? ((await base .orderBy(sql`LOWER(${extensions.id})`, extensions.id) @@ -180,13 +193,11 @@ export class ExtensionsDatabase { return this.notFound(id); } - const releases = sortReleasesDescending( - parseJSON(row.releases, []) - ); + const releases = sortReleasesDescending(legacyReleases(row.releases)); return { data: { latestRelease: releases[0] ?? null, - license: parseJSON(row.license, { name: "" }) + license: legacyLicense(row.license) }, error: null }; @@ -194,10 +205,10 @@ export class ExtensionsDatabase { } function parseExtensionRow(row: ExtensionRow): Extension { - const releases = parseJSON(row.releases, []); + const releases = legacyReleases(row.releases); return { id: row.id, - type: row.type as Extension["type"], + type: (row.type === "module" ? "mod" : row.type) as Extension["type"], name: row.name, description: row.description, author: { @@ -212,11 +223,37 @@ function parseExtensionRow(row: ExtensionRow): Extension { } as Author, releases: sortReleasesDescending(releases), website: row.website, - license: parseJSON(row.license, { name: "" }), - icon_url: row.iconUrl ?? "", + license: legacyLicense(row.license), + // Package-relative manifest icons have no public URL in the v1 contract. + icon_url: + row.iconUrl && /^https?:\/\//i.test(row.iconUrl) ? row.iconUrl : "", readme: row.readme, source: parseJSON(row.source, { type: "custom", repo: "" }), version: row.version, download_url: row.downloadUrl }; } + +function legacyLicense(stored: string): { name: string; URL?: string } { + const value = parseJSON(stored, { + name: "" + }); + return typeof value === "string" ? { name: value } : value; +} + +function legacyReleases(stored: string): Release[] { + return parseJSON>( + stored, + [] + ).map((release) => { + const { requires, ...legacy } = release; + if (legacy.min_fossbilling_version) return legacy; + let minimum = "0.0.0"; + try { + minimum = minVersion(requires?.fossbilling ?? "*")?.version ?? minimum; + } catch { + /* Composer-only ranges have no v1 representation. */ + } + return { ...legacy, min_fossbilling_version: minimum }; + }); +} diff --git a/src/services/extensions/v1/index.ts b/src/services/extensions/v1/index.ts index 205b6753..4b3aad47 100644 --- a/src/services/extensions/v1/index.ts +++ b/src/services/extensions/v1/index.ts @@ -27,7 +27,7 @@ extensionsV1.use("/*", trimTrailingSlash()); // The assembled list is ~126KB of readme + release history per entry, rebuilt // from a full-table D1 read (per the documented legacy contract) on every -// request - yet catalogue content changes at human speed. Cache the serialized +// request - yet extension content changes at human speed. Cache the serialized // response body in CACHE_KV and let concurrent cold misses share one build per // isolate; a cached response also freezes the unpaginated query's otherwise // nondeterministic row order. diff --git a/src/services/extensions/v1/interfaces.ts b/src/services/extensions/v1/interfaces.ts index 6c2533a6..79429486 100644 --- a/src/services/extensions/v1/interfaces.ts +++ b/src/services/extensions/v1/interfaces.ts @@ -1,12 +1,13 @@ -import { EXTENSION_TYPES } from "../v2/schemas/extensions"; - export type Extension = { id: string; - // Derived from v2 rather than restated: both describe the same - // extensions.type column (v1/database.ts already reads v2's db/schema), so a - // new extension type would otherwise pass v2's runtime validator while v1's - // type silently disagreed. - type: (typeof EXTENSION_TYPES)[number]; + type: + | "mod" + | "theme" + | "payment-gateway" + | "server-manager" + | "domain-registrar" + | "hook" + | "translation"; name: string; description: string; author: Author; diff --git a/src/services/extensions/v1/list-cache.ts b/src/services/extensions/v1/list-cache.ts index e5bb4b9e..c3111824 100644 --- a/src/services/extensions/v1/list-cache.ts +++ b/src/services/extensions/v1/list-cache.ts @@ -1,5 +1,5 @@ // Cache-key surface for the v1 list body cache (see index.ts). Kept separate -// so extensions/v2's revalidateCatalogue can invalidate it after catalogue +// so extensions/v2's revalidateExtensions can invalidate it after extension directory // mutations without importing the v1 app module (which would drag the whole // Hono app into v2's import graph for one function). import { EXTENSION_TYPES } from "../v2/schemas/extensions"; diff --git a/src/services/extensions/v2/README.md b/src/services/extensions/v2/README.md index 5ddd81d4..524b2d70 100644 --- a/src/services/extensions/v2/README.md +++ b/src/services/extensions/v2/README.md @@ -2,7 +2,7 @@ **Base Path:** `/extensions/v2` -Self-service extension publishing, developer-profile ownership, moderation, and public catalogue browsing. +Self-service extension publishing, developer-profile ownership, moderation, and browsing published extensions. This service owns the complete Extensions domain and its `DB_EXTENSIONS` schema. The separate Extensions site keeps OIDC/session state but reaches this domain through the generated HTTPS API client; it must not bind or migrate `DB_EXTENSIONS`. @@ -13,16 +13,34 @@ Endpoints are not listed here. The service publishes its own contract: - **OpenAPI document:** `GET /extensions/v2/docs/openapi.json` - **Reference UI:** `GET /extensions/v2/docs` +## Manifest contract + +New v2 writes use the [FOSSBilling manifest v1 fields](https://github.com/FOSSBilling/modules/blob/main/manifest.schema.json): `manifest_version: 1`, `type: "module"` (replacing `mod`; former `hook` entries become modules), `author: {name, url?}`, a string `license`, optional `homepage_url` (replacing `website`), and `requires: {fossbilling, php, modules?}`. IDs allow lowercase letters, digits, underscores, and hyphens, up to 64 characters. Names are limited to 100 characters, descriptions to 500, and versions use SemVer without a `v` prefix. + +`icon_url` is an optional path relative to the package root, as in `icon.svg`; it cannot be an absolute URL, absolute path, or contain `..` or backslashes. Consumers resolve package assets against their trusted distribution source. Historical icons may still be absolute URLs. + +API publishing fields `readme`, `source`, `download_url`, and `releases` remain alongside the manifest metadata. Each release carries its own `requires` object instead of `min_fossbilling_version`. Constraints are retained verbatim, including Composer ranges and module dependencies; package CI remains responsible for semantic constraint/license validation and confirming that icon files exist. The API adds size bounds to metadata (author names: 200 characters; license and requirement strings: 1,000; URLs and icon paths: 2,048). + +`author` is package attribution. `developer` identifies the authenticated owner and remains controlled by the API; an author field never grants ownership. + +Migration `0029_align_manifest.sql` backfills published metadata from the owning developer and release requirements. The selected release's former minimum becomes `>=minimum`; missing requirements, including historical PHP requirements, become `*` to indicate that no constraint was recorded. License identifiers use the old `spdx_id` when available, otherwise the display name. Physical `website` storage remains for v1 compatibility. Reviewed and pending revision bodies retain their original audit content; old pending revisions must be resubmitted using the new contract before approval. Existing IDs and published historical content remain readable without being forced through new-write validation. + +Apply the migration and API deployment in a controlled cutover: the old Worker expects object licenses, while the new Worker requires the added metadata column. + +This changes the v2 request/response contract: regenerate the Extensions site's client and update CI publishers together with the API rollout. The modules publisher must stop translating `module` to `mod` and send the manifest fields directly. Package-relative icons use v1's empty-string fallback. The v1 adapter continues to return `mod`, `website`, object licenses, and minimum release versions; complex ranges are reduced to the earliest SemVer version when representable, with `0.0.0` for Composer-only ranges that v1 cannot express. + +The frontend revalidation tags are `extensions` and `developers`, shared with the Extensions site. Deploy the API and site tag change together. New code calls `revalidateExtensions(c)`. + ## Lifecycle An extension exists from `POST /extensions`, not from approval. There is no separate submission record. -- `POST /extensions` creates the record and reserves its id. It stays out of catalogues until its first revision is approved. +- `POST /extensions` creates the record and reserves its id. It stays out of public listings until its first revision is approved. - `PUT /extensions/{id}` proposes an edit. Published content is unchanged until approval. One unreviewed revision per extension at a time. - `DELETE /extensions/{id}` withdraws an unpublished extension and releases its id. Published extensions cannot be withdrawn by owners. - `POST /extensions/{id}/revisions/{revisionId}/approve` publishes. `reject` leaves published content untouched. - `POST /extensions/{id}/delist` (moderator-only) hides a published extension for cause without deleting content or history. `POST /extensions/{id}/relist` restores it. Both accept `?notify=false`. -- `POST /extensions/{id}/moderator-correct` (moderator-only) fixes live catalogue corruption without waiting for the author. Writes an already-approved revision, leaves `published_at` and ownership untouched. Requires a published, listed extension with no pending revision (409 otherwise). No `?notify` option and no author email; the fix is recorded in revision history. +- `POST /extensions/{id}/moderator-correct` (moderator-only) fixes live extension content corruption without waiting for the author. Writes an already-approved revision, leaves `published_at` and ownership untouched. Requires a published, listed extension with no pending revision (409 otherwise). No `?notify` option and no author email; the fix is recorded in revision history. An edit cannot rename an extension or move it to another developer. A user owns at most one developer profile, so no request body names one. @@ -32,7 +50,7 @@ An edit cannot rename an extension or move it to another developer. A user owns `GET /extensions?scope=` selects the projection: -- `public` (default, anonymous allowed): published catalogue cards. +- `public` (default, anonymous allowed): published extension cards. - `mine`: caller's own extensions. - `all` (moderator only): every extension, filterable by `status` (`published`, `delisted`, `unpublished`) and `q` (case-insensitive id substring). @@ -44,7 +62,7 @@ Owner/moderator extension reads return four independent fields, not a derived st | `null` | set | rejected | Rejected, already resubmitted | | `null` | `null` | rejected | Rejected; edit and resubmit | | set | `null` | approved | Live, no unreviewed edit | -| set | `null` | `null` | Live, adopted from pre-v2 catalogue | +| set | `null` | `null` | Live, adopted from pre-v2 directory | | set | set | either | Live, edit awaiting review | `delisted` is orthogonal to all three: it hides the row from public reads without touching them. Adopted rows have no revisions; a live extension with no review history is normal. @@ -55,7 +73,7 @@ Anonymous detail reads are cacheable (`Cache-Control: public`); detail reads tha ## Pagination -All v2 lists use opaque keyset cursors: `?limit=` (1–100, default 50) with `?cursor=` from the previous page's `pagination.next_cursor`. Envelope is always `pagination: {next_cursor, has_more}`. An invalid cursor returns `INVALID_CURSOR` (422); restart from the first page. List items omit `readme` and `releases`; fetch `GET /extensions/{id}` for detail. +All v2 lists use opaque keyset cursors: `?limit=` (1–100, default 50) with `?cursor=` from the previous page's `pagination.next_cursor`. Envelope is always `pagination: {next_cursor, has_more}`. An invalid cursor returns `INVALID_CURSOR` (422); restart from the first page. List items omit `readme`, `releases`, and `requires`; fetch `GET /extensions/{id}` for detail. ## Authentication @@ -98,7 +116,7 @@ Recipients: developer `contact_email`, falling back to the owning account's emai ## Limits and budgets -Content writes (`POST /extensions`, `PUT /extensions/{id}`, moderator-correct) count the raw request stream before JSON parsing. Raw limit is **512 KiB** (`413 BODY_TOO_LARGE`); normalized JSON content limit is **256 KiB**. New slug ids are at most 200 characters. +Content writes (`POST /extensions`, `PUT /extensions/{id}`, moderator-correct) count the raw request stream before JSON parsing. Raw limit is **512 KiB** (`413 BODY_TOO_LARGE`); normalized JSON content limit is **256 KiB**. New slug ids are at most 64 characters. `EXTENSION_WRITE_RATE_LIMITER` paces IP and account attempts at 60/minute, including validation failures. It is approximate edge pacing, not the durable quota. Missing pacing fails closed (`503 ADMISSION_UNAVAILABLE`). diff --git a/src/services/extensions/v2/db/extensions.ts b/src/services/extensions/v2/db/extensions.ts index 223e1b45..11eb4c70 100644 --- a/src/services/extensions/v2/db/extensions.ts +++ b/src/services/extensions/v2/db/extensions.ts @@ -27,6 +27,7 @@ import { Release, Repository } from "../schemas/extensions"; +import { Manifest, manifestMetadata } from "../schemas/manifest"; import { PublicDeveloper } from "../schemas/developers"; export const MAX_PENDING_REVISIONS_PER_USER = 10; @@ -61,12 +62,17 @@ const CONTENT_COLUMNS = { license: sql< string | null >`CASE WHEN length(CAST(${extensions.license} AS BLOB)) <= ${MAX_CONTENT_BYTES} AND json_valid(${extensions.license}) THEN - json_patch(json_object('name', substr(json_extract(${extensions.license}, '$.name'), 1, 100)), - json_patch(CASE WHEN json_type(${extensions.license}, '$.spdx_id') = 'text' - THEN json_object('spdx_id', substr(json_extract(${extensions.license}, '$.spdx_id'), 1, 100)) ELSE '{}' END, - CASE WHEN json_type(${extensions.license}, '$.URL') = 'text' AND length(json_extract(${extensions.license}, '$.URL')) <= 2048 - THEN json_object('URL', json_extract(${extensions.license}, '$.URL')) ELSE '{}' END)) - ELSE '{"name":"Unavailable"}' END`, + json_quote(substr(CASE WHEN json_type(${extensions.license}) = 'text' THEN json_extract(${extensions.license}, '$') + ELSE COALESCE(json_extract(${extensions.license}, '$.spdx_id'), json_extract(${extensions.license}, '$.name'), 'Unavailable') END, 1, 1000)) + ELSE '"Unavailable"' END`, + manifest: sql< + string | null + >`CASE WHEN length(CAST(${extensions.manifest} AS BLOB)) <= ${MAX_CONTENT_BYTES} AND json_valid(${extensions.manifest}) THEN + json_object('manifest_version', 1, 'author', json_patch( + json_object('name', substr(json_extract(${extensions.manifest}, '$.author.name'), 1, 200)), + CASE WHEN json_type(${extensions.manifest}, '$.author.url') = 'text' AND length(json_extract(${extensions.manifest}, '$.author.url')) <= 2048 + THEN json_object('url', json_extract(${extensions.manifest}, '$.author.url')) ELSE '{}' END)) + ELSE NULL END`, iconUrl: sql< string | null >`CASE WHEN length(${extensions.iconUrl}) <= 2048 THEN ${extensions.iconUrl} ELSE NULL END`, @@ -86,6 +92,9 @@ const CONTENT_COLUMNS = { // Detail preserves stored fields; the overall size guard prevents oversized reads. const DETAIL_CONTENT_COLUMNS = { + manifest: sql< + string | null + >`CASE WHEN ${publishedBytes} <= ${MAX_CONTENT_BYTES} THEN ${extensions.manifest} ELSE NULL END`, type: sql< string | null >`CASE WHEN ${publishedBytes} <= ${MAX_CONTENT_BYTES} THEN ${extensions.type} ELSE NULL END`, @@ -179,7 +188,7 @@ const REVIEW_COLUMNS = { reviewedAt: REVIEWED.reviewedAt }; -// The owner list drops the same two large published fields the catalogue does, +// The owner list omits README, releases, and requirements, like public lists, // and the pending revision's stored content (up to 256 KiB per row) with it. const OWNED_LIST_COLUMNS = { id: extensions.id, @@ -244,6 +253,7 @@ interface DeveloperRow { // filter sufficient: a published row cannot be missing any of them. That // constraint is what makes the non-null types here sound. interface PublishedRow extends DeveloperRow { + manifest: string | null; id: string; type: string; name: string; @@ -513,7 +523,7 @@ export class ExtensionsDatabase { }; } - // Catalogue totals for the admin tabs, using the same published / + // Extension directory totals for the admin tabs, using the same published / // delisted / unpublished predicates as listForModeration so the badges // can never disagree with what each tab lists. One conditional-SUM // query rather than four COUNTs. @@ -855,7 +865,7 @@ export class ExtensionsDatabase { } // Withdrawing is only offered while an extension has never been published: - // once it is in the catalogue, consumers pin its id and removing it is a + // once it is in the extension directory, consumers pin its id and removing it is a // moderator's decision, not the owner's. async withdraw( id: string, @@ -932,7 +942,7 @@ export class ExtensionsDatabase { } // A moderator's decision to pull an already-published extension from the - // catalogue for cause - its upstream source disappearing, for example. + // extension directory for cause - its upstream source disappearing, for example. // Content and history are kept (unlike withdraw(), which deletes a // never-published row outright), so an owner can still see why and a // moderator can re-list it later without the developer resubmitting from @@ -1024,7 +1034,7 @@ export class ExtensionsDatabase { } // Inverse of delist(): restores a delisted-but-published extension to the - // catalogue. Content and history are untouched; only the delist markers + // extension directory. Content and history are untouched; only the delist markers // are cleared. `AND delisted_at IS NOT NULL` makes this a single atomic // check-and-set mirroring delist(). The actor guard re-checks moderator // status inside the statement (not just activity): requireModerator() @@ -1119,7 +1129,7 @@ export class ExtensionsDatabase { }; } - // Moderator correction of live catalogue content (api#251): a truncated + // Moderator correction of live extension content (api#251): a truncated // readme or broken link the owner should not have to resubmit to fix. One // D1 batch() like approve(): the first statement inserts an // already-approved revision row (submitted_by and reviewer both the @@ -1187,7 +1197,7 @@ export class ExtensionsDatabase { sql: `UPDATE extensions SET type = ?, name = ?, description = ?, releases = ?, website = ?, license = ?, icon_url = ?, readme = ?, source = ?, version = ?, - download_url = ?, + download_url = ?, manifest = ?, published_revision_id = ?, updated_at = CURRENT_TIMESTAMP WHERE changes() = 1 AND LOWER(id) = LOWER(?)`, @@ -1196,13 +1206,14 @@ export class ExtensionsDatabase { valid.name, valid.description, JSON.stringify(valid.releases), - valid.website, + valid.homepage_url ?? "", JSON.stringify(valid.license), valid.icon_url ?? null, valid.readme, JSON.stringify(valid.source), valid.version, valid.download_url, + JSON.stringify(manifestMetadata(valid)), revisionId, id ] @@ -1350,20 +1361,17 @@ function parseDeveloper(row: DeveloperRow): PublicDeveloper { }; } -// Shared by both parsers so the catalogue card and the detail view can never -// disagree about the embedded developer. +// Keep URLs on list cards within the response size bound. function cardUrl(url: string | null): string | null { return url && url.length <= 2048 ? url : null; } -function cardLicense(stored: string | null): License { - const license = parseJSON(stored ?? "", { name: "Unavailable" }); - license.name = - typeof license.name === "string" && license.name - ? license.name.slice(0, 100) - : "Unavailable"; - if (license.URL && license.URL.length > 2048) delete license.URL; - return license; +function parseLicense(stored: string | null): License { + const value = parseJSON( + stored ?? "", + "Unavailable" + ); + return typeof value === "string" ? value : (value.spdx_id ?? value.name); } function cardSource(stored: string | null): Repository { @@ -1378,9 +1386,10 @@ function cardSource(stored: string | null): Repository { return source; } -// The field subset both the public catalogue rows and the owner listing rows +// The field subset both the public extension listing rows and the owner listing rows // project, in their nullable table form. interface ListContentFields { + manifest: string | null; type: string | null; name: string | null; description: string | null; @@ -1393,7 +1402,7 @@ interface ListContentFields { } // The card/detail content projection shared by parseListRow (public -// catalogue) and the owned listing's published payload. Shaping in one place +// extension directory) and the owned listing's published payload. Shaping in one place // keeps the two payload producers from drifting; the non-null assertions are // sound because every caller only reaches here for rows whose published_at // is set, where extensions_published_content_check guarantees presence. @@ -1402,17 +1411,38 @@ interface ListContentFields { // already performs in SQL. They are belt-and-braces for legacy rows written // before that truncation existed - not dead code, and not to be "simplified" // away without an audit of pre-migration rows. -function listContent(row: ListContentFields, card: boolean) { +function listContent( + row: ListContentFields, + card: true +): Omit; +function listContent( + row: ListContentFields, + card: false +): Omit; +function listContent( + row: ListContentFields, + card: boolean +): + | Omit + | Omit { + const metadata = parseManifestMetadata(row.manifest); return { - type: row.type as ExtensionContent["type"], + ...metadata, + requires: card ? undefined : metadata.requires, + author: card + ? { ...metadata.author, name: metadata.author.name.slice(0, 200) } + : metadata.author, + type: (row.type === "mod" + ? "module" + : row.type) as ExtensionContent["type"], name: card ? (row.name as string).slice(0, 120) : (row.name as string), description: card ? (row.description as string).slice(0, 4000) : (row.description as string), - website: card ? cardUrl(row.website) : row.website, + homepage_url: (card ? cardUrl(row.website) : row.website) || undefined, license: card - ? cardLicense(row.license) - : parseJSON(row.license as string, { name: "" }), + ? parseLicense(row.license).slice(0, 1000) + : parseLicense(row.license), icon_url: (card ? cardUrl(row.iconUrl) : row.iconUrl) ?? undefined, source: card ? cardSource(row.source) @@ -1449,20 +1479,22 @@ function keysetPage( }; } -function parseListRow(row: PublishedListRow, card = true): ExtensionListItem { +function parseListRow(row: PublishedListRow): ExtensionListItem { return { id: row.id, - ...listContent(row, card), + ...listContent(row, true), developer: parseDeveloper(row) }; } // The detail view is the list projection plus the two large fields the -// catalogue query deliberately omits. +// extension directory query deliberately omits. function parseRow(row: PublishedRow): Extension { return { - ...parseListRow(row, false), - website: row.website, + id: row.id, + developer: parseDeveloper(row), + ...listContent(row, false), + homepage_url: row.website || undefined, download_url: row.downloadUrl, readme: row.readme, releases: boundedReleases(parseJSON(row.releases, [])) @@ -1502,7 +1534,7 @@ function parseOwnedRow(row: OwnedRow): OwnedExtension { published: row.publishedAt ? { ...listContent(row, false), - website: row.website as string, + homepage_url: row.website || undefined, download_url: row.downloadUrl as string, readme: row.readme as string, releases: boundedReleases( @@ -1527,7 +1559,7 @@ export function parseContent(stored: string | null): StoredExtensionContent { const content = parseJSON(stored ?? "", {}); return { ...content, - releases: boundedReleases(content.releases ?? []) + releases: boundedReleases(content.releases ?? [], false) }; } @@ -1535,7 +1567,7 @@ export class LegacyContentError extends Error {} // Legacy bodies bypassed today's schema. Bound the collection and tag work // before semver sorting, while retaining support for partial historical content. -function boundedReleases(value: unknown): Release[] { +function boundedReleases(value: unknown, normalize = true): Release[] { if ( !Array.isArray(value) || value.length > 100 || @@ -1550,7 +1582,21 @@ function boundedReleases(value: unknown): Release[] { throw new LegacyContentError( "Legacy release collection exceeds safe read bounds" ); - return sortReleasesDescending(value as Release[]); + if (!normalize) return sortReleasesDescending(value as Release[]); + return sortReleasesDescending( + value.map((release) => { + const { min_fossbilling_version, ...rest } = release; + return { + ...rest, + requires: rest.requires ?? { + fossbilling: min_fossbilling_version + ? `>=${min_fossbilling_version}` + : "*", + php: "*" + } + }; + }) as Release[] + ); } export function oversizedContent(): DatabaseResult { @@ -1563,3 +1609,13 @@ export function oversizedContent(): DatabaseResult { } }; } + +function parseManifestMetadata( + stored: string | null +): Pick { + return parseJSON(stored ?? "", { + manifest_version: 1, + author: { name: "Unknown" }, + requires: { fossbilling: "*", php: "*" } + }); +} diff --git a/src/services/extensions/v2/db/migrations/0029_align_manifest.sql b/src/services/extensions/v2/db/migrations/0029_align_manifest.sql new file mode 100644 index 00000000..7e0a8cd6 --- /dev/null +++ b/src/services/extensions/v2/db/migrations/0029_align_manifest.sql @@ -0,0 +1,245 @@ +-- Preserve physical website/license storage for v1, while v2 exposes manifest names. +-- Reviewed revision bodies remain an audit log in their original format. +ALTER TABLE extensions ADD COLUMN manifest TEXT; +--> statement-breakpoint +DROP TRIGGER resource_extensions_insert; +--> statement-breakpoint +CREATE TRIGGER resource_extensions_insert AFTER INSERT ON extensions BEGIN + INSERT INTO extension_resource_usage(scope,subject,bytes,extensions,revisions) + SELECT scope, subject, COALESCE(length(CAST(NEW.manifest AS BLOB)),0) + + COALESCE(length(CAST(NEW.type AS BLOB)),0) + + COALESCE(length(CAST(NEW.name AS BLOB)),0) + + COALESCE(length(CAST(NEW.description AS BLOB)),0) + + COALESCE(length(CAST(NEW.releases AS BLOB)),0) + + COALESCE(length(CAST(NEW.website AS BLOB)),0) + + COALESCE(length(CAST(NEW.license AS BLOB)),0) + + COALESCE(length(CAST(NEW.icon_url AS BLOB)),0) + + COALESCE(length(CAST(NEW.readme AS BLOB)),0) + + COALESCE(length(CAST(NEW.source AS BLOB)),0) + + COALESCE(length(CAST(NEW.version AS BLOB)),0) + + COALESCE(length(CAST(NEW.download_url AS BLOB)),0), 1, 0 + FROM ( + SELECT 'global' AS scope, 'all' AS subject + UNION ALL SELECT 'account', COALESCE(NEW.created_by,'legacy') + UNION ALL SELECT 'developer', NEW.developer_id + ) WHERE true + ON CONFLICT(scope,subject) DO UPDATE SET + bytes=bytes+excluded.bytes, + extensions=extensions+excluded.extensions, + revisions=revisions+excluded.revisions; + SELECT CASE WHEN EXISTS ( + SELECT 1 FROM extension_resource_usage + WHERE ((scope='account' AND subject=COALESCE(NEW.created_by,'legacy')) + OR (scope='developer' AND subject=NEW.developer_id)) + AND (bytes>26214400 OR extensions>100 OR revisions>1000) + ) THEN RAISE(ABORT,'extension_resource_quota') END; +END; +--> statement-breakpoint +DROP TRIGGER resource_extensions_update; +--> statement-breakpoint +CREATE TRIGGER resource_extensions_update AFTER UPDATE OF type,name,description,releases,website,license,icon_url,readme,source,version,download_url,manifest ON extensions BEGIN + UPDATE extension_resource_usage + SET bytes = bytes + (COALESCE(length(CAST(NEW.manifest AS BLOB)),0) + + COALESCE(length(CAST(NEW.type AS BLOB)),0) + + COALESCE(length(CAST(NEW.name AS BLOB)),0) + + COALESCE(length(CAST(NEW.description AS BLOB)),0) + + COALESCE(length(CAST(NEW.releases AS BLOB)),0) + + COALESCE(length(CAST(NEW.website AS BLOB)),0) + + COALESCE(length(CAST(NEW.license AS BLOB)),0) + + COALESCE(length(CAST(NEW.icon_url AS BLOB)),0) + + COALESCE(length(CAST(NEW.readme AS BLOB)),0) + + COALESCE(length(CAST(NEW.source AS BLOB)),0) + + COALESCE(length(CAST(NEW.version AS BLOB)),0) + + COALESCE(length(CAST(NEW.download_url AS BLOB)),0)) - (COALESCE(length(CAST(OLD.manifest AS BLOB)),0) + + COALESCE(length(CAST(OLD.type AS BLOB)),0) + + COALESCE(length(CAST(OLD.name AS BLOB)),0) + + COALESCE(length(CAST(OLD.description AS BLOB)),0) + + COALESCE(length(CAST(OLD.releases AS BLOB)),0) + + COALESCE(length(CAST(OLD.website AS BLOB)),0) + + COALESCE(length(CAST(OLD.license AS BLOB)),0) + + COALESCE(length(CAST(OLD.icon_url AS BLOB)),0) + + COALESCE(length(CAST(OLD.readme AS BLOB)),0) + + COALESCE(length(CAST(OLD.source AS BLOB)),0) + + COALESCE(length(CAST(OLD.version AS BLOB)),0) + + COALESCE(length(CAST(OLD.download_url AS BLOB)),0)) + WHERE (scope='global' AND subject='all') + OR (scope='account' AND subject=COALESCE(NEW.created_by,'legacy')) + OR (scope='developer' AND subject=NEW.developer_id); +END; +--> statement-breakpoint +DROP TRIGGER sync_extensions_bytes_insert; +--> statement-breakpoint +CREATE TRIGGER sync_extensions_bytes_insert AFTER INSERT ON extensions +WHEN NEW.published_bytes != (COALESCE(length(CAST(NEW.manifest AS BLOB)),0) + + COALESCE(length(CAST(NEW.type AS BLOB)),0) + + COALESCE(length(CAST(NEW.name AS BLOB)),0) + + COALESCE(length(CAST(NEW.description AS BLOB)),0) + + COALESCE(length(CAST(NEW.releases AS BLOB)),0) + + COALESCE(length(CAST(NEW.website AS BLOB)),0) + + COALESCE(length(CAST(NEW.license AS BLOB)),0) + + COALESCE(length(CAST(NEW.icon_url AS BLOB)),0) + + COALESCE(length(CAST(NEW.readme AS BLOB)),0) + + COALESCE(length(CAST(NEW.source AS BLOB)),0) + + COALESCE(length(CAST(NEW.version AS BLOB)),0) + + COALESCE(length(CAST(NEW.download_url AS BLOB)),0)) BEGIN + UPDATE extensions SET published_bytes=(COALESCE(length(CAST(NEW.manifest AS BLOB)),0) + + COALESCE(length(CAST(NEW.type AS BLOB)),0) + + COALESCE(length(CAST(NEW.name AS BLOB)),0) + + COALESCE(length(CAST(NEW.description AS BLOB)),0) + + COALESCE(length(CAST(NEW.releases AS BLOB)),0) + + COALESCE(length(CAST(NEW.website AS BLOB)),0) + + COALESCE(length(CAST(NEW.license AS BLOB)),0) + + COALESCE(length(CAST(NEW.icon_url AS BLOB)),0) + + COALESCE(length(CAST(NEW.readme AS BLOB)),0) + + COALESCE(length(CAST(NEW.source AS BLOB)),0) + + COALESCE(length(CAST(NEW.version AS BLOB)),0) + + COALESCE(length(CAST(NEW.download_url AS BLOB)),0)) + WHERE id=NEW.id; +END; +--> statement-breakpoint +DROP TRIGGER sync_extensions_bytes_update; +--> statement-breakpoint +CREATE TRIGGER sync_extensions_bytes_update AFTER UPDATE OF type,name,description,releases,website,license,icon_url,readme,source,version,download_url,manifest,published_bytes ON extensions +WHEN NEW.published_bytes != (COALESCE(length(CAST(NEW.manifest AS BLOB)),0) + + COALESCE(length(CAST(NEW.type AS BLOB)),0) + + COALESCE(length(CAST(NEW.name AS BLOB)),0) + + COALESCE(length(CAST(NEW.description AS BLOB)),0) + + COALESCE(length(CAST(NEW.releases AS BLOB)),0) + + COALESCE(length(CAST(NEW.website AS BLOB)),0) + + COALESCE(length(CAST(NEW.license AS BLOB)),0) + + COALESCE(length(CAST(NEW.icon_url AS BLOB)),0) + + COALESCE(length(CAST(NEW.readme AS BLOB)),0) + + COALESCE(length(CAST(NEW.source AS BLOB)),0) + + COALESCE(length(CAST(NEW.version AS BLOB)),0) + + COALESCE(length(CAST(NEW.download_url AS BLOB)),0)) BEGIN + UPDATE extensions SET published_bytes=(COALESCE(length(CAST(NEW.manifest AS BLOB)),0) + + COALESCE(length(CAST(NEW.type AS BLOB)),0) + + COALESCE(length(CAST(NEW.name AS BLOB)),0) + + COALESCE(length(CAST(NEW.description AS BLOB)),0) + + COALESCE(length(CAST(NEW.releases AS BLOB)),0) + + COALESCE(length(CAST(NEW.website AS BLOB)),0) + + COALESCE(length(CAST(NEW.license AS BLOB)),0) + + COALESCE(length(CAST(NEW.icon_url AS BLOB)),0) + + COALESCE(length(CAST(NEW.readme AS BLOB)),0) + + COALESCE(length(CAST(NEW.source AS BLOB)),0) + + COALESCE(length(CAST(NEW.version AS BLOB)),0) + + COALESCE(length(CAST(NEW.download_url AS BLOB)),0)) + WHERE id=NEW.id; +END; +--> statement-breakpoint +DROP TRIGGER resource_extensions_delete; +--> statement-breakpoint +CREATE TRIGGER resource_extensions_delete AFTER DELETE ON extensions BEGIN + UPDATE extension_resource_usage + SET bytes=bytes-(COALESCE(length(CAST(OLD.manifest AS BLOB)),0) + COALESCE(length(CAST(OLD.type AS BLOB)),0) + + COALESCE(length(CAST(OLD.name AS BLOB)),0) + + COALESCE(length(CAST(OLD.description AS BLOB)),0) + + COALESCE(length(CAST(OLD.releases AS BLOB)),0) + + COALESCE(length(CAST(OLD.website AS BLOB)),0) + + COALESCE(length(CAST(OLD.license AS BLOB)),0) + + COALESCE(length(CAST(OLD.icon_url AS BLOB)),0) + + COALESCE(length(CAST(OLD.readme AS BLOB)),0) + + COALESCE(length(CAST(OLD.source AS BLOB)),0) + + COALESCE(length(CAST(OLD.version AS BLOB)),0) + + COALESCE(length(CAST(OLD.download_url AS BLOB)),0)), extensions=extensions-1 + WHERE (scope='global' AND subject='all') + OR (scope='account' AND subject=COALESCE(OLD.created_by,'legacy')) + OR (scope='developer' AND subject=OLD.developer_id); +END; +--> statement-breakpoint +UPDATE extensions SET + type = CASE WHEN type IN ('mod','hook') THEN 'module' ELSE type END, + license = CASE WHEN json_valid(license) THEN + CASE WHEN json_type(license) = 'object' THEN json_quote(COALESCE(json_extract(license,'$.spdx_id'),json_extract(license,'$.name'),'Unavailable')) ELSE license END + ELSE json_quote(COALESCE(license,'Unavailable')) END, + manifest = json_object( + 'manifest_version', 1, + 'author', json((SELECT CASE WHEN url IS NULL OR url = '' THEN json_object('name',name) ELSE json_object('name',name,'url',url) END FROM developers WHERE developers.id=extensions.developer_id)), + 'requires', json_object('fossbilling', COALESCE( + (SELECT '>=' || json_extract(value,'$.min_fossbilling_version') FROM json_each(CASE WHEN json_valid(extensions.releases) THEN extensions.releases ELSE '[]' END) + WHERE json_extract(value,'$.min_fossbilling_version') IS NOT NULL + ORDER BY CASE WHEN json_extract(value,'$.tag') IN (extensions.version,'v'||extensions.version) THEN 0 ELSE 1 END, key LIMIT 1), '*'), 'php','*') + ) +WHERE published_at IS NOT NULL; + +--> statement-breakpoint +-- Preserve every release entry while replacing the lossy minimum-version field. +UPDATE extensions SET releases = ( + SELECT json_group_array(json(json_set(json_remove(value,'$.min_fossbilling_version'), '$.requires', + json(CASE WHEN json_type(value,'$.requires') = 'object' THEN json_extract(value,'$.requires') + ELSE json_object('fossbilling',CASE WHEN json_extract(value,'$.min_fossbilling_version') IS NOT NULL + THEN '>=' || json_extract(value,'$.min_fossbilling_version') ELSE '*' END,'php','*') END)))) + FROM json_each(extensions.releases) +) WHERE published_at IS NOT NULL AND json_valid(releases) AND json_type(releases)='array'; +--> statement-breakpoint +DROP INDEX idx_extensions_catalogue_order; +--> statement-breakpoint +DROP INDEX idx_extensions_type_catalogue_order; +--> statement-breakpoint +CREATE INDEX idx_extensions_public_order ON extensions (lower(id),id) WHERE published_at IS NOT NULL AND delisted_at IS NULL; +--> statement-breakpoint +CREATE INDEX idx_extensions_type_public_order ON extensions (type,lower(id),id) WHERE published_at IS NOT NULL AND delisted_at IS NULL; + +--> statement-breakpoint +-- Enforce growth quotas again after the metadata backfill; pre-existing +-- over-quota collections remain intact and cannot grow through new writes. +DROP TRIGGER resource_extensions_update; +--> statement-breakpoint +CREATE TRIGGER resource_extensions_update AFTER UPDATE OF type,name,description,releases,website,license,icon_url,readme,source,version,download_url,manifest ON extensions BEGIN + UPDATE extension_resource_usage + SET bytes = bytes + (COALESCE(length(CAST(NEW.manifest AS BLOB)),0) + + COALESCE(length(CAST(NEW.type AS BLOB)),0) + + COALESCE(length(CAST(NEW.name AS BLOB)),0) + + COALESCE(length(CAST(NEW.description AS BLOB)),0) + + COALESCE(length(CAST(NEW.releases AS BLOB)),0) + + COALESCE(length(CAST(NEW.website AS BLOB)),0) + + COALESCE(length(CAST(NEW.license AS BLOB)),0) + + COALESCE(length(CAST(NEW.icon_url AS BLOB)),0) + + COALESCE(length(CAST(NEW.readme AS BLOB)),0) + + COALESCE(length(CAST(NEW.source AS BLOB)),0) + + COALESCE(length(CAST(NEW.version AS BLOB)),0) + + COALESCE(length(CAST(NEW.download_url AS BLOB)),0)) - (COALESCE(length(CAST(OLD.manifest AS BLOB)),0) + + COALESCE(length(CAST(OLD.type AS BLOB)),0) + + COALESCE(length(CAST(OLD.name AS BLOB)),0) + + COALESCE(length(CAST(OLD.description AS BLOB)),0) + + COALESCE(length(CAST(OLD.releases AS BLOB)),0) + + COALESCE(length(CAST(OLD.website AS BLOB)),0) + + COALESCE(length(CAST(OLD.license AS BLOB)),0) + + COALESCE(length(CAST(OLD.icon_url AS BLOB)),0) + + COALESCE(length(CAST(OLD.readme AS BLOB)),0) + + COALESCE(length(CAST(OLD.source AS BLOB)),0) + + COALESCE(length(CAST(OLD.version AS BLOB)),0) + + COALESCE(length(CAST(OLD.download_url AS BLOB)),0)) + WHERE (scope='global' AND subject='all') + OR (scope='account' AND subject=COALESCE(NEW.created_by,'legacy')) + OR (scope='developer' AND subject=NEW.developer_id); + SELECT CASE WHEN (COALESCE(length(CAST(NEW.manifest AS BLOB)),0) + + COALESCE(length(CAST(NEW.type AS BLOB)),0) + + COALESCE(length(CAST(NEW.name AS BLOB)),0) + + COALESCE(length(CAST(NEW.description AS BLOB)),0) + + COALESCE(length(CAST(NEW.releases AS BLOB)),0) + + COALESCE(length(CAST(NEW.website AS BLOB)),0) + + COALESCE(length(CAST(NEW.license AS BLOB)),0) + + COALESCE(length(CAST(NEW.icon_url AS BLOB)),0) + + COALESCE(length(CAST(NEW.readme AS BLOB)),0) + + COALESCE(length(CAST(NEW.source AS BLOB)),0) + + COALESCE(length(CAST(NEW.version AS BLOB)),0) + + COALESCE(length(CAST(NEW.download_url AS BLOB)),0)) > (COALESCE(length(CAST(OLD.manifest AS BLOB)),0) + + COALESCE(length(CAST(OLD.type AS BLOB)),0) + + COALESCE(length(CAST(OLD.name AS BLOB)),0) + + COALESCE(length(CAST(OLD.description AS BLOB)),0) + + COALESCE(length(CAST(OLD.releases AS BLOB)),0) + + COALESCE(length(CAST(OLD.website AS BLOB)),0) + + COALESCE(length(CAST(OLD.license AS BLOB)),0) + + COALESCE(length(CAST(OLD.icon_url AS BLOB)),0) + + COALESCE(length(CAST(OLD.readme AS BLOB)),0) + + COALESCE(length(CAST(OLD.source AS BLOB)),0) + + COALESCE(length(CAST(OLD.version AS BLOB)),0) + + COALESCE(length(CAST(OLD.download_url AS BLOB)),0)) AND EXISTS ( + SELECT 1 FROM extension_resource_usage + WHERE ((scope='account' AND subject=COALESCE(NEW.created_by,'legacy')) + OR (scope='developer' AND subject=NEW.developer_id)) + AND (bytes>26214400 OR extensions>100 OR revisions>1000) + ) + THEN RAISE(ABORT,'extension_resource_quota') END; +END; diff --git a/src/services/extensions/v2/db/migrations/meta/0029_snapshot.json b/src/services/extensions/v2/db/migrations/meta/0029_snapshot.json new file mode 100644 index 00000000..ba8162d8 --- /dev/null +++ b/src/services/extensions/v2/db/migrations/meta/0029_snapshot.json @@ -0,0 +1,1369 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "522dcadb-834f-4249-9e4e-611e233a50d2", + "prevId": "417f6ce6-9d14-46b0-ba84-234dee5e4cc8", + "tables": { + "claim_verification_budgets": { + "name": "claim_verification_budgets", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_claim_verification_budgets_expiry": { + "name": "idx_claim_verification_budgets_expiry", + "columns": ["expires_at"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "developer_claims": { + "name": "developer_claims", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "developer_id": { + "name": "developer_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "claimant_id": { + "name": "claimant_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "note": { + "name": "note", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "review_note": { + "name": "review_note", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reviewer_id": { + "name": "reviewer_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + }, + "reviewed_at": { + "name": "reviewed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "github_org_verified": { + "name": "github_org_verified", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "github_verification_note": { + "name": "github_verification_note", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "idx_developer_claims_developer": { + "name": "idx_developer_claims_developer", + "columns": ["developer_id"], + "isUnique": false + }, + "idx_developer_claims_claimant": { + "name": "idx_developer_claims_claimant", + "columns": ["claimant_id"], + "isUnique": false + }, + "idx_developer_claims_pending_unique": { + "name": "idx_developer_claims_pending_unique", + "columns": ["developer_id", "claimant_id"], + "isUnique": true, + "where": "\"developer_claims\".\"status\" = 'pending'" + }, + "idx_developer_claims_pending_queue": { + "name": "idx_developer_claims_pending_queue", + "columns": ["created_at"], + "isUnique": false, + "where": "\"developer_claims\".\"status\" = 'pending'" + } + }, + "foreignKeys": { + "developer_claims_developer_id_developers_id_fk": { + "name": "developer_claims_developer_id_developers_id_fk", + "tableFrom": "developer_claims", + "tableTo": "developers", + "columnsFrom": ["developer_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "developer_claims_claimant_id_users_id_fk": { + "name": "developer_claims_claimant_id_users_id_fk", + "tableFrom": "developer_claims", + "tableTo": "users", + "columnsFrom": ["claimant_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "developer_claims_reviewer_id_users_id_fk": { + "name": "developer_claims_reviewer_id_users_id_fk", + "tableFrom": "developer_claims", + "tableTo": "users", + "columnsFrom": ["reviewer_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": { + "developer_claims_status_check": { + "name": "developer_claims_status_check", + "value": "\"developer_claims\".\"status\" IN ('pending', 'approved', 'rejected')" + }, + "developer_claims_github_org_verified_check": { + "name": "developer_claims_github_org_verified_check", + "value": "\"developer_claims\".\"github_org_verified\" IN (0, 1)" + } + } + }, + "developer_history": { + "name": "developer_history", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "developer_id": { + "name": "developer_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "changed_by": { + "name": "changed_by", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "changed_at": { + "name": "changed_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + } + }, + "indexes": { + "idx_developer_history_developer_changed_at": { + "name": "idx_developer_history_developer_changed_at", + "columns": ["developer_id", "changed_at"], + "isUnique": false + }, + "idx_developer_history_account_changed_at": { + "name": "idx_developer_history_account_changed_at", + "columns": ["changed_by", "changed_at"], + "isUnique": false + } + }, + "foreignKeys": { + "developer_history_changed_by_users_id_fk": { + "name": "developer_history_changed_by_users_id_fk", + "tableFrom": "developer_history", + "tableTo": "users", + "columnsFrom": ["changed_by"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "developer_transfers": { + "name": "developer_transfers", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "developer_id": { + "name": "developer_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token_hash": { + "name": "token_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + }, + "expires_at": { + "name": "expires_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "accepted_by": { + "name": "accepted_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "accepted_at": { + "name": "accepted_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "revoked_at": { + "name": "revoked_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "idx_developer_transfers_token": { + "name": "idx_developer_transfers_token", + "columns": ["token_hash"], + "isUnique": true + }, + "idx_developer_transfers_pending": { + "name": "idx_developer_transfers_pending", + "columns": ["developer_id"], + "isUnique": true, + "where": "\"developer_transfers\".\"accepted_at\" IS NULL AND \"developer_transfers\".\"revoked_at\" IS NULL" + } + }, + "foreignKeys": { + "developer_transfers_developer_id_developers_id_fk": { + "name": "developer_transfers_developer_id_developers_id_fk", + "tableFrom": "developer_transfers", + "tableTo": "developers", + "columnsFrom": ["developer_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "developer_transfers_created_by_users_id_fk": { + "name": "developer_transfers_created_by_users_id_fk", + "tableFrom": "developer_transfers", + "tableTo": "users", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "developer_transfers_accepted_by_users_id_fk": { + "name": "developer_transfers_accepted_by_users_id_fk", + "tableFrom": "developer_transfers", + "tableTo": "users", + "columnsFrom": ["accepted_by"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "developers": { + "name": "developers", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "owner_user_id": { + "name": "owner_user_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "approved_at": { + "name": "approved_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'1970-01-01T00:00:00.000Z'" + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'1970-01-01T00:00:00.000Z'" + }, + "avatar_url": { + "name": "avatar_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "contact_email": { + "name": "contact_email", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "ownership_epoch": { + "name": "ownership_epoch", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "content_revision": { + "name": "content_revision", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "profile_generation": { + "name": "profile_generation", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "approved_revision": { + "name": "approved_revision", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "approved_by": { + "name": "approved_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "github_org_verified": { + "name": "github_org_verified", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "github_verification_note": { + "name": "github_verification_note", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "github_verified_at": { + "name": "github_verified_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "github_url_verified": { + "name": "github_url_verified", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "url_check_cooldown_until": { + "name": "url_check_cooldown_until", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "idx_developers_owner_unique": { + "name": "idx_developers_owner_unique", + "columns": ["owner_user_id"], + "isUnique": true + }, + "idx_developers_approved": { + "name": "idx_developers_approved", + "columns": ["approved_at"], + "isUnique": false + } + }, + "foreignKeys": { + "developers_owner_user_id_users_id_fk": { + "name": "developers_owner_user_id_users_id_fk", + "tableFrom": "developers", + "tableTo": "users", + "columnsFrom": ["owner_user_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": { + "developers_ownership_epoch_check": { + "name": "developers_ownership_epoch_check", + "value": "\"developers\".\"ownership_epoch\" >= 1" + }, + "developers_content_revision_check": { + "name": "developers_content_revision_check", + "value": "\"developers\".\"content_revision\" >= 1" + }, + "developers_github_org_verified_check": { + "name": "developers_github_org_verified_check", + "value": "\"developers\".\"github_org_verified\" IN (0, 1)" + }, + "developers_github_url_verified_check": { + "name": "developers_github_url_verified_check", + "value": "\"developers\".\"github_url_verified\" = 1" + } + } + }, + "extension_resource_usage": { + "name": "extension_resource_usage", + "columns": { + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "subject": { + "name": "subject", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes": { + "name": "bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "extensions": { + "name": "extensions", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "revisions": { + "name": "revisions", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + } + }, + "indexes": { + "idx_extension_resource_usage_subject": { + "name": "idx_extension_resource_usage_subject", + "columns": ["scope", "subject"], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "extension_revisions": { + "name": "extension_revisions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "extension_id": { + "name": "extension_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "developer_id": { + "name": "developer_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "submitted_by": { + "name": "submitted_by", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content_bytes": { + "name": "content_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "summary_name": { + "name": "summary_name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "summary_version": { + "name": "summary_version", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "summary_description": { + "name": "summary_description", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "content_readable": { + "name": "content_readable", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "content_hash": { + "name": "content_hash", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "compacted_at": { + "name": "compacted_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reviewer_id": { + "name": "reviewer_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "review_note": { + "name": "review_note", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + }, + "reviewed_at": { + "name": "reviewed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "ownership_epoch": { + "name": "ownership_epoch", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + } + }, + "indexes": { + "idx_extension_revisions_retention": { + "name": "idx_extension_revisions_retention", + "columns": ["compacted_at", "reviewed_at", "id", "status"], + "isUnique": false + }, + "idx_extension_revisions_submitted_by": { + "name": "idx_extension_revisions_submitted_by", + "columns": ["submitted_by"], + "isUnique": false + }, + "idx_extension_revisions_developer": { + "name": "idx_extension_revisions_developer", + "columns": ["developer_id"], + "isUnique": false + }, + "idx_extension_revisions_pending": { + "name": "idx_extension_revisions_pending", + "columns": ["extension_id"], + "isUnique": true, + "where": "\"extension_revisions\".\"status\" = 'pending'" + }, + "idx_extension_revisions_extension_page": { + "name": "idx_extension_revisions_extension_page", + "columns": ["extension_id", "\"created_at\" desc", "\"id\" desc"], + "isUnique": false + }, + "idx_extension_revisions_submitter_page": { + "name": "idx_extension_revisions_submitter_page", + "columns": ["submitted_by", "\"created_at\" desc", "\"id\" desc"], + "isUnique": false + }, + "idx_extension_revisions_queue_page": { + "name": "idx_extension_revisions_queue_page", + "columns": ["status", "created_at", "id"], + "isUnique": false + }, + "idx_extension_revisions_reviewed": { + "name": "idx_extension_revisions_reviewed", + "columns": ["extension_id", "reviewed_at"], + "isUnique": false, + "where": "\"extension_revisions\".\"status\" IN ('approved', 'rejected')" + }, + "idx_extension_revisions_submitter_pending": { + "name": "idx_extension_revisions_submitter_pending", + "columns": ["submitted_by"], + "isUnique": false, + "where": "\"extension_revisions\".\"status\" = 'pending'" + } + }, + "foreignKeys": { + "extension_revisions_extension_id_extensions_id_fk": { + "name": "extension_revisions_extension_id_extensions_id_fk", + "tableFrom": "extension_revisions", + "tableTo": "extensions", + "columnsFrom": ["extension_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "extension_revisions_submitted_by_users_id_fk": { + "name": "extension_revisions_submitted_by_users_id_fk", + "tableFrom": "extension_revisions", + "tableTo": "users", + "columnsFrom": ["submitted_by"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "extension_revisions_reviewer_id_users_id_fk": { + "name": "extension_revisions_reviewer_id_users_id_fk", + "tableFrom": "extension_revisions", + "tableTo": "users", + "columnsFrom": ["reviewer_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": { + "extension_revisions_status_check": { + "name": "extension_revisions_status_check", + "value": "\"extension_revisions\".\"status\" IN ('pending', 'approved', 'rejected')" + }, + "extension_revisions_ownership_epoch_check": { + "name": "extension_revisions_ownership_epoch_check", + "value": "\"extension_revisions\".\"ownership_epoch\" >= 1" + } + } + }, + "extension_write_events": { + "name": "extension_write_events", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "developer_id": { + "name": "developer_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "occurred_at": { + "name": "occurred_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_extension_write_events_account": { + "name": "idx_extension_write_events_account", + "columns": ["account_id", "occurred_at"], + "isUnique": false + }, + "idx_extension_write_events_developer": { + "name": "idx_extension_write_events_developer", + "columns": ["developer_id", "occurred_at"], + "isUnique": false + }, + "idx_extension_write_events_time": { + "name": "idx_extension_write_events_time", + "columns": ["occurred_at"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "extensions": { + "name": "extensions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "developer_id": { + "name": "developer_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "published_bytes": { + "name": "published_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "published_at": { + "name": "published_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "published_revision_id": { + "name": "published_revision_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "releases": { + "name": "releases", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "website": { + "name": "website", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "license": { + "name": "license", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "icon_url": { + "name": "icon_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "readme": { + "name": "readme", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "version": { + "name": "version", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "download_url": { + "name": "download_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + }, + "delisted_at": { + "name": "delisted_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "delist_reason": { + "name": "delist_reason", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "manifest": { + "name": "manifest", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "idx_extensions_published_revision": { + "name": "idx_extensions_published_revision", + "columns": ["published_revision_id"], + "isUnique": false + }, + "idx_extensions_id_nocase": { + "name": "idx_extensions_id_nocase", + "columns": ["lower(\"id\")"], + "isUnique": true + }, + "idx_extensions_developer_order": { + "name": "idx_extensions_developer_order", + "columns": ["developer_id", "lower(\"id\")", "id"], + "isUnique": false + }, + "idx_extensions_public_order": { + "name": "idx_extensions_public_order", + "columns": ["lower(\"id\")", "id"], + "isUnique": false, + "where": "\"extensions\".\"published_at\" IS NOT NULL AND \"extensions\".\"delisted_at\" IS NULL" + }, + "idx_extensions_type_public_order": { + "name": "idx_extensions_type_public_order", + "columns": ["type", "lower(\"id\")", "id"], + "isUnique": false, + "where": "\"extensions\".\"published_at\" IS NOT NULL AND \"extensions\".\"delisted_at\" IS NULL" + } + }, + "foreignKeys": { + "extensions_developer_id_developers_id_fk": { + "name": "extensions_developer_id_developers_id_fk", + "tableFrom": "extensions", + "tableTo": "developers", + "columnsFrom": ["developer_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": { + "extensions_published_content_check": { + "name": "extensions_published_content_check", + "value": "\"extensions\".\"published_at\" IS NULL OR (\n \"extensions\".\"type\" IS NOT NULL AND \"extensions\".\"name\" IS NOT NULL AND\n \"extensions\".\"description\" IS NOT NULL AND \"extensions\".\"releases\" IS NOT NULL AND\n \"extensions\".\"website\" IS NOT NULL AND \"extensions\".\"license\" IS NOT NULL AND\n \"extensions\".\"readme\" IS NOT NULL AND \"extensions\".\"source\" IS NOT NULL AND\n \"extensions\".\"version\" IS NOT NULL AND \"extensions\".\"download_url\" IS NOT NULL\n )" + } + } + }, + "m2m_publisher_clients": { + "name": "m2m_publisher_clients", + "columns": { + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "developer_id": { + "name": "developer_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "can_publish": { + "name": "can_publish", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "ownership_epoch": { + "name": "ownership_epoch", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + }, + "revoked_at": { + "name": "revoked_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "idx_m2m_publisher_clients_developer": { + "name": "idx_m2m_publisher_clients_developer", + "columns": ["developer_id"], + "isUnique": false + } + }, + "foreignKeys": { + "m2m_publisher_clients_developer_id_developers_id_fk": { + "name": "m2m_publisher_clients_developer_id_developers_id_fk", + "tableFrom": "m2m_publisher_clients", + "tableTo": "developers", + "columnsFrom": ["developer_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": { + "m2m_publisher_clients_can_publish_check": { + "name": "m2m_publisher_clients_can_publish_check", + "value": "\"m2m_publisher_clients\".\"can_publish\" IN (0, 1)" + }, + "m2m_publisher_clients_ownership_epoch_check": { + "name": "m2m_publisher_clients_ownership_epoch_check", + "value": "\"m2m_publisher_clients\".\"ownership_epoch\" >= 1" + } + } + }, + "users": { + "name": "users", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "email_verified": { + "name": "email_verified", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "picture": { + "name": "picture", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "is_moderator": { + "name": "is_moderator", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "github_login": { + "name": "github_login", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "github_orgs": { + "name": "github_orgs", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "github_orgs_expires_at": { + "name": "github_orgs_expires_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": { + "idx_extension_revisions_extension_page": { + "columns": { + "\"created_at\" desc": { + "isExpression": true + }, + "\"id\" desc": { + "isExpression": true + } + } + }, + "idx_extension_revisions_submitter_page": { + "columns": { + "\"created_at\" desc": { + "isExpression": true + }, + "\"id\" desc": { + "isExpression": true + } + } + }, + "idx_extensions_id_nocase": { + "columns": { + "lower(\"id\")": { + "isExpression": true + } + } + }, + "idx_extensions_developer_order": { + "columns": { + "lower(\"id\")": { + "isExpression": true + } + } + }, + "idx_extensions_public_order": { + "columns": { + "lower(\"id\")": { + "isExpression": true + } + } + }, + "idx_extensions_type_public_order": { + "columns": { + "lower(\"id\")": { + "isExpression": true + } + } + } + } + } +} diff --git a/src/services/extensions/v2/db/migrations/meta/_journal.json b/src/services/extensions/v2/db/migrations/meta/_journal.json index dd6922fa..456cece5 100644 --- a/src/services/extensions/v2/db/migrations/meta/_journal.json +++ b/src/services/extensions/v2/db/migrations/meta/_journal.json @@ -106,6 +106,13 @@ "when": 1793554482651, "tag": "0028_m2m_publisher_clients", "breakpoints": true + }, + { + "idx": 29, + "version": "6", + "when": 1793793600000, + "tag": "0029_align_manifest", + "breakpoints": true } ] } diff --git a/src/services/extensions/v2/db/revisions.ts b/src/services/extensions/v2/db/revisions.ts index 26b6b66b..0fe75f04 100644 --- a/src/services/extensions/v2/db/revisions.ts +++ b/src/services/extensions/v2/db/revisions.ts @@ -1,3 +1,4 @@ +import { manifestMetadata } from "../schemas/manifest"; import { MAX_CONTENT_BYTES } from "../resource-limits"; import { and, asc, desc, eq, gt, lt, or, sql, SQL } from "drizzle-orm"; import { DatabaseError, DatabaseResult } from "../../../../lib/interfaces"; @@ -686,7 +687,7 @@ export class ExtensionRevisionsDatabase { // The stored content is deliberately validated again here rather than // trusted from submission time. A revision can predate the current content // rules - migration 0021 carries legacy submissions through verbatim - and - // publishing is the one boundary where the public catalogue's stricter + // publishing is the one boundary where the public extension listing's stricter // contract has to hold. This is the same reason the pre-0021 code // re-checked reserved ids at approval. const parsed = ExtensionContentSchema.safeParse(revision.content); @@ -720,7 +721,7 @@ export class ExtensionRevisionsDatabase { extensionId: string ): D1PreparedStatement { // published_at is COALESCEd rather than overwritten: it records when the - // extension first entered the catalogue, and updated_at carries the + // extension first entered the extension directory, and updated_at carries the // "changed just now" signal. The WHERE changes() = 1 gates the publish on // the immediately-preceding claim statement (SQLite's per-connection // changes() function) inside the same D1 batch transaction - a race or @@ -731,7 +732,7 @@ export class ExtensionRevisionsDatabase { sql: `UPDATE extensions SET type = ?, name = ?, description = ?, releases = ?, website = ?, license = ?, icon_url = ?, readme = ?, source = ?, version = ?, - download_url = ?, + download_url = ?, manifest = ?, published_at = COALESCE(published_at, CURRENT_TIMESTAMP), published_revision_id = ?, updated_at = CURRENT_TIMESTAMP @@ -741,13 +742,14 @@ export class ExtensionRevisionsDatabase { content.name, content.description, JSON.stringify(content.releases), - content.website, + content.homepage_url ?? "", JSON.stringify(content.license), content.icon_url ?? null, content.readme, JSON.stringify(content.source), content.version, content.download_url, + JSON.stringify(manifestMetadata(content)), revisionId, extensionId ] diff --git a/src/services/extensions/v2/db/schema.ts b/src/services/extensions/v2/db/schema.ts index 3fd85d74..58319588 100644 --- a/src/services/extensions/v2/db/schema.ts +++ b/src/services/extensions/v2/db/schema.ts @@ -32,7 +32,7 @@ export const users = sqliteTable("users", { // An extension record exists from the moment a developer creates it, before // any moderator has seen it. The content columns are therefore the *published* // projection and are NULL until the first revision is approved (migration -// 0021); published_at is the marker the public catalogue filters on, and the +// 0021); published_at is the marker the public extension listing filters on, and the // CHECK below is what keeps "published" from ever meaning "half a row". // // The column was author_id until migration 0021. It never had to be: the only @@ -52,9 +52,10 @@ export const extensions = sqliteTable( // a FK: extension_revisions.extension_id already points the other way, and // a second FK between the same two tables would make them mutually // dependent for both inserts and the SQLite table rebuilds that migrations - // need. NULL for rows adopted from the pre-v2 catalogue, which were never + // need. NULL for rows adopted from the pre-v2 directory, which were never // published through a revision. publishedRevisionId: text("published_revision_id"), + manifest: text("manifest"), type: text("type"), name: text("name"), description: text("description"), @@ -73,7 +74,7 @@ export const extensions = sqliteTable( .notNull() .default(sql`CURRENT_TIMESTAMP`), // A moderator's decision to pull an already-published extension from the - // catalogue for cause (its upstream source disappearing, for example). + // extension directory for cause (its upstream source disappearing, for example). // Distinct from never having been published and from an owner's // pre-publish withdraw(): published_at and delisted_at can both be set at // once, so the fact that this was live and then pulled is never lost, and @@ -101,13 +102,13 @@ export const extensions = sqliteTable( ), // These two are partial: every read that uses them filters on // published_at IS NOT NULL AND delisted_at IS NULL, so an unpublished or - // delisted row would only bloat the index the catalogue scans. - index("idx_extensions_catalogue_order") + // delisted row would only bloat the index the extension directory scans. + index("idx_extensions_public_order") .on(sql`lower(${table.id})`, table.id) .where( sql`${table.publishedAt} IS NOT NULL AND ${table.delistedAt} IS NULL` ), - index("idx_extensions_type_catalogue_order") + index("idx_extensions_type_public_order") .on(table.type, sql`lower(${table.id})`, table.id) .where( sql`${table.publishedAt} IS NOT NULL AND ${table.delistedAt} IS NULL` @@ -133,9 +134,9 @@ export const extensions = sqliteTable( ] ); -// Legacy catalogue table renamed from authors to developers by migration +// Legacy extension directory table renamed from authors to developers by migration // 0008. The API owns the full table now; type/name/url are the original -// catalogue fields and the remaining columns were added by the v2 migrations. +// extension directory fields and the remaining columns were added by the v2 migrations. // bio (added in 0003) was dropped in 0010 and is intentionally absent here. export const developers = sqliteTable( "developers", diff --git a/src/services/extensions/v2/revalidate.ts b/src/services/extensions/v2/revalidate.ts index 83b1adba..ffa6cacd 100644 --- a/src/services/extensions/v2/revalidate.ts +++ b/src/services/extensions/v2/revalidate.ts @@ -2,24 +2,24 @@ import { Context } from "hono"; import { getPlatform } from "../../../lib/middleware"; import { invalidateListCache } from "../v1/list-cache"; -// The extensions site route-caches its public catalogue pages behind +// The extensions site route-caches its public extension pages behind // Cloudflare-CDN-Cache-Control (maxAge + stale-while-revalidate). Those // windows are the correctness floor for freshness; this purge is the latency // optimization that makes api-side mutations visible within a second or two // instead of after a cache window. // -// Catalogue-mutating endpoints must call revalidateCatalogue(c) after a +// Extension-mutating endpoints must call revalidateExtensions(c) after a // successful write (see AGENTS.md). The call is fire-and-forget: failures // are logged and swallowed, because the mutation has already succeeded and // the cache windows bound any resulting staleness. // -// The v1 list body cache (../v1/list-cache) hangs off the same catalogue, so +// The v1 list body cache (../v1/list-cache) hangs off the same extension directory, so // it is invalidated here too - same fire-and-forget trade: a failed delete // merely leaves the list stale until its own short TTL expires. -const REVALIDATE_TAGS = ["catalogue", "developers"]; +const REVALIDATE_TAGS = ["extensions", "developers"]; const REVALIDATE_URL = "https://extensions.fossbilling.org/api/revalidate"; -export function revalidateCatalogue( +export function revalidateExtensions( c: Context<{ Bindings: CloudflareBindings }> ): void { invalidateListCache(c.env.CACHE_KV, (p) => c.executionCtx.waitUntil(p)); diff --git a/src/services/extensions/v2/routes/developer-profiles.ts b/src/services/extensions/v2/routes/developer-profiles.ts index 4ed44d1a..5c70b874 100644 --- a/src/services/extensions/v2/routes/developer-profiles.ts +++ b/src/services/extensions/v2/routes/developer-profiles.ts @@ -33,7 +33,7 @@ import { toPublicDeveloper } from "../schemas/developers"; import { DeveloperProfilesDatabase } from "../db/developer-profiles"; -import { revalidateCatalogue } from "../revalidate"; +import { revalidateExtensions } from "../revalidate"; import { UsersDatabase } from "../db/users"; import { ExtensionsV2App } from "./app"; @@ -229,8 +229,8 @@ export function registerDeveloperProfileRoutes(app: ExtensionsV2App): void { return response; } // Profile edits apply immediately (no moderation staging) and change - // catalogue-visible fields (developer name/URL), so purge here too. - if (changed) revalidateCatalogue(c); + // publicly visible fields (developer name/URL), so purge here too. + if (changed) revalidateExtensions(c); return c.json({ result: data }, 200); }); @@ -279,7 +279,7 @@ export function registerDeveloperProfileRoutes(app: ExtensionsV2App): void { // extensions, which are still public content — purge for consistency // (it is a no-op today while no cached route carries 'developers', but // keeps this endpoint correct if that ever changes). - revalidateCatalogue(c); + revalidateExtensions(c); return c.json({ result: data }, 200); }); diff --git a/src/services/extensions/v2/routes/moderation.ts b/src/services/extensions/v2/routes/moderation.ts index 4fe2996f..55f71f2b 100644 --- a/src/services/extensions/v2/routes/moderation.ts +++ b/src/services/extensions/v2/routes/moderation.ts @@ -38,7 +38,7 @@ import { DeveloperProfilesDatabase } from "../db/developer-profiles"; import { ExtensionsDatabase } from "../db/extensions"; import { ExtensionRevisionsDatabase } from "../db/revisions"; import { notifyAuthor } from "./notify"; -import { revalidateCatalogue } from "../revalidate"; +import { revalidateExtensions } from "../revalidate"; import { ExtensionsV2App } from "./app"; export function registerModerationRoutes(app: ExtensionsV2App): void { @@ -192,14 +192,14 @@ export function registerModerationRoutes(app: ExtensionsV2App): void { }); // Distinct from reject: reject leaves a pending edit unpublished, delist - // pulls an already-published extension out of the catalogue entirely. See + // pulls an already-published extension out of the extension directory entirely. See // ExtensionsDatabase.delist() for why content and history are kept rather // than cleared. const delistRoute = createRoute({ method: "post", path: "/extensions/{id}/delist", tags: ["Moderation"], - summary: "Remove a published extension from the public catalogue", + summary: "Remove a published extension from the public extension listing", security: [{ Bearer: [] }], middleware: [requireModerator()] as const, request: { @@ -223,7 +223,7 @@ export function registerModerationRoutes(app: ExtensionsV2App): void { } }, description: - "Extension removed from the public catalogue. Its content and " + + "Extension removed from the public extension listing. Its content and " + "history are kept, and its owner can still see and edit it." }, ...authErrorResponses(ModeratorForbiddenResponse), @@ -263,7 +263,7 @@ export function registerModerationRoutes(app: ExtensionsV2App): void { method: "post", path: "/extensions/{id}/relist", tags: ["Moderation"], - summary: "Restore a delisted extension to the public catalogue", + summary: "Restore a delisted extension to the public extension listing", security: [{ Bearer: [] }], middleware: [requireModerator()] as const, request: { @@ -287,7 +287,7 @@ export function registerModerationRoutes(app: ExtensionsV2App): void { } }, description: - "Extension restored to the public catalogue. Content and history unchanged." + "Extension restored to the public extension listing. Content and history unchanged." }, ...authErrorResponses(ModeratorForbiddenResponse), 404: errorResponse("No such extension"), @@ -399,7 +399,7 @@ export function registerModerationRoutes(app: ExtensionsV2App): void { const status = statusFromContentWriteError(error?.code); return c.json(errorBody(error, "Unable to correct extension"), status); } - revalidateCatalogue(c); + revalidateExtensions(c); return c.json( { result: { diff --git a/src/services/extensions/v2/routes/notify.ts b/src/services/extensions/v2/routes/notify.ts index c1de2eac..f5c4706f 100644 --- a/src/services/extensions/v2/routes/notify.ts +++ b/src/services/extensions/v2/routes/notify.ts @@ -6,17 +6,17 @@ import { sendModerationNotification, type ModerationNotifyInput } from "../email/notify"; -import { revalidateCatalogue } from "../revalidate"; +import { revalidateExtensions } from "../revalidate"; // Shared tail of routes that notify authors after moderation writes: purge -// the CDN-cached catalogue pages, then — unless ?notify=false opted out — +// the CDN-cached extension pages, then — unless ?notify=false opted out — // dispatch the author notification off the response path. Returns the // `notified` flag the routes report in their response bodies. // (moderatorCorrect also mutates but deliberately sends no email, so it -// calls revalidateCatalogue() directly instead of this helper.) +// calls revalidateExtensions() directly instead of this helper.) // // A shared route-layer helper rather than a register*Routes module, like -// routes/errors.ts; it touches the Hono context and revalidateCatalogue, so +// routes/errors.ts; it touches the Hono context and revalidateExtensions, so // it deliberately lives beside its call sites rather than in email/. export async function notifyAuthor( c: Context, @@ -24,7 +24,7 @@ export async function notifyAuthor( query: { notify?: string }, input: ModerationNotifyInput ): Promise { - revalidateCatalogue(c); + revalidateExtensions(c); if (!notifyRequested(query)) return false; return sendModerationNotification(getPlatform(c), extDb, input, (p) => c.executionCtx.waitUntil(p) diff --git a/src/services/extensions/v2/routes/owner-extensions.ts b/src/services/extensions/v2/routes/owner-extensions.ts index 31523e8b..483ec02a 100644 --- a/src/services/extensions/v2/routes/owner-extensions.ts +++ b/src/services/extensions/v2/routes/owner-extensions.ts @@ -42,7 +42,7 @@ import { } from "../db/extensions"; import { ExtensionRevisionsDatabase } from "../db/revisions"; import { UsersDatabase } from "../db/users"; -import { revalidateCatalogue } from "../revalidate"; +import { revalidateExtensions } from "../revalidate"; import { ExtensionsV2App } from "./app"; // Both history and detail authorize against current account/profile state; @@ -97,7 +97,7 @@ export function registerOwnerExtensionsRoutes(app: ExtensionsV2App): void { 201: { content: { "application/json": { schema: AcceptedRevisionSchema } }, description: - "Extension created. It holds the id immediately but stays out of the public catalogue until a moderator approves the revision." + "Extension created. It holds the id immediately but stays out of the public extension listing until a moderator approves the revision." }, ...authErrorResponses(), 403: { @@ -315,7 +315,7 @@ export function registerOwnerExtensionsRoutes(app: ExtensionsV2App): void { statusFromWriteErrorCode(error?.code) ); } - revalidateCatalogue(c); + revalidateExtensions(c); return c.json({ result: { id: data.id, deleted: true as const } }, 200); }); diff --git a/src/services/extensions/v2/routes/public-extensions.ts b/src/services/extensions/v2/routes/public-extensions.ts index a5a4f3fd..64b22712 100644 --- a/src/services/extensions/v2/routes/public-extensions.ts +++ b/src/services/extensions/v2/routes/public-extensions.ts @@ -24,7 +24,7 @@ export function registerPublicExtensionsRoutes(app: ExtensionsV2App): void { path: "/extensions", tags: ["Extensions"], summary: - "List extensions: published catalogue (scope=public), caller's own (scope=mine), or every extension (scope=all, moderator)", + "List extensions: published extension directory (scope=public), caller's own (scope=mine), or every extension (scope=all, moderator)", security: [{ Bearer: [] }], middleware: [optionalAuth()] as const, request: { query: UnifiedExtensionListQuerySchema }, @@ -36,7 +36,7 @@ export function registerPublicExtensionsRoutes(app: ExtensionsV2App): void { } }, description: - "Extensions matching the scope: catalogue cards for scope=public, owned rows for scope=mine/all" + "Extensions matching the scope: extension cards for scope=public, owned rows for scope=mine/all" }, 401: errorResponse("Invalid bearer token"), 403: errorResponse( diff --git a/src/services/extensions/v2/schemas/extensions.ts b/src/services/extensions/v2/schemas/extensions.ts index 6ebf55b1..4c09c342 100644 --- a/src/services/extensions/v2/schemas/extensions.ts +++ b/src/services/extensions/v2/schemas/extensions.ts @@ -1,23 +1,14 @@ import { MAX_CONTENT_BYTES } from "../resource-limits"; import { z } from "@hono/zod-openapi"; -import SPDX_LICENSE_IDS from "spdx-license-ids/index.json"; import { - cursorPaginationFields, - httpUrl, - lowercaseId, - PaginationSchema -} from "./common"; + ManifestSchema, + MANIFEST_EXTENSION_TYPES, + ManifestRequirementsSchema +} from "./manifest"; +import { cursorPaginationFields, httpUrl, PaginationSchema } from "./common"; import { PublicDeveloperSchema } from "./developers"; -export const EXTENSION_TYPES = [ - "mod", - "theme", - "payment-gateway", - "server-manager", - "domain-registrar", - "hook", - "translation" -] as const; +export const EXTENSION_TYPES = MANIFEST_EXTENSION_TYPES; export const ReleaseSchema = z .object({ @@ -25,7 +16,7 @@ export const ReleaseSchema = z date: z.string().min(1).max(64), download_url: httpUrl(), changelog_url: httpUrl().optional(), - min_fossbilling_version: z.string().min(1).max(100) + requires: ManifestRequirementsSchema }) .strict() .openapi("Release"); @@ -42,31 +33,7 @@ export const RepositorySchema = z export type Repository = z.infer; -// `spdx-license-ids` ships only the current (non-deprecated) identifiers — -// https://github.com/jslicense/spdx-license-ids — so submitters are steered -// toward the license SPDX currently recommends, not a retired alias. -const spdxLicenseId = () => - z - .string() - .refine((value) => SPDX_LICENSE_IDS.includes(value), { - message: "must be a current (non-deprecated) SPDX license identifier" - }) - .openapi({ - description: - "A current SPDX license identifier (https://spdx.org/licenses/). " + - "Omitted for custom or proprietary licenses.", - example: "MIT" - }); - -export const LicenseSchema = z - .object({ - name: z.string().min(1).max(100), - spdx_id: spdxLicenseId().optional(), - URL: httpUrl().optional() - }) - .strict() - .openapi("License"); - +export const LicenseSchema = ManifestSchema.shape.license; export type License = z.infer; // Everything about an extension that a developer can edit. Excludes `id`, @@ -74,16 +41,19 @@ export type License = z.infer; // revision stores and what moderators approve. export const ExtensionContentSchema = z .object({ - type: z.enum(EXTENSION_TYPES), - name: z.string().min(1).max(120), - description: z.string().min(1).max(4000), + manifest_version: ManifestSchema.shape.manifest_version, + type: ManifestSchema.shape.type, + name: ManifestSchema.shape.name, + description: ManifestSchema.shape.description, + author: ManifestSchema.shape.author, + requires: ManifestSchema.shape.requires, releases: z.array(ReleaseSchema).min(1).max(100), - website: httpUrl(), + homepage_url: ManifestSchema.shape.homepage_url, license: LicenseSchema, - icon_url: httpUrl().optional(), + icon_url: ManifestSchema.shape.icon_url, readme: z.string().min(1).max(100_000), source: RepositorySchema, - version: z.string().min(1).max(100), + version: ManifestSchema.shape.version, download_url: httpUrl() }) .openapi("ExtensionContent"); @@ -102,10 +72,12 @@ export type ExtensionContent = z.infer; // a record written under rules that no longer exist. Field types and upper // bounds are kept, since those still say something true about the shape. // Nothing is weakened for publication - approve() revalidates against the -// strict schema before anything reaches the catalogue. +// strict schema before anything reaches the extension directory. // SQLite length() counts Unicode code points. Use the same bound for legacy // reads, with a 200-code-unit ceiling so supplementary characters still fit. const StoredReleaseSchema = ReleaseSchema.extend({ + requires: ManifestRequirementsSchema.optional(), + min_fossbilling_version: z.string().optional(), tag: z .string() .min(1) @@ -115,6 +87,20 @@ const StoredReleaseSchema = ReleaseSchema.extend({ }); export const StoredExtensionContentSchema = ExtensionContentSchema.extend({ + type: z.enum([...EXTENSION_TYPES, "mod", "hook"]), + name: z.string().min(1).max(120), + description: z.string().min(1).max(4000), + version: z.string().min(1).max(100), + icon_url: z.union([ManifestSchema.shape.icon_url, httpUrl()]).optional(), + website: httpUrl().optional(), + license: z.union([ + LicenseSchema, + z.object({ + name: z.string(), + spdx_id: z.string().optional(), + URL: httpUrl().optional() + }) + ]), releases: z.array(StoredReleaseSchema).max(100) }) .partial() @@ -139,7 +125,7 @@ function refineContentSize(content: unknown, ctx: z.RefinementCtx): void { // POST /extensions. The id is chosen once here and is immutable afterwards. // No developer field: a user owns at most one profile, so the server knows it. export const ExtensionCreateSchema = ExtensionContentSchema.extend({ - id: lowercaseId("extension").max(200) + id: ManifestSchema.shape.id }) .strict() .superRefine(refineContentSize) @@ -152,23 +138,39 @@ export const ExtensionUpdateSchema = ExtensionContentSchema.strict() .superRefine(refineContentSize) .openapi("ExtensionUpdate"); -// The public projection: published content plus the developer that owns it. -export const ExtensionSchema = ExtensionContentSchema.extend({ +// Published reads also cover pre-manifest rows, whose display fields and +// icon URLs may predate the current input constraints. +const PublishedContentSchema = ExtensionContentSchema.extend({ + name: z.string().min(1).max(120), + description: z.string().min(1).max(4000), + version: z.string().min(1).max(100), + license: z.string().min(1), + icon_url: z.string().optional() +}); + +// The public projection: published package attribution plus its current owner. +export const ExtensionSchema = PublishedContentSchema.extend({ id: z.string(), developer: PublicDeveloperSchema }).openapi("Extension"); export type Extension = z.infer; -// Catalogue cards do not need the potentially large README or every historic -// release. Consumers can fetch those fields from GET /extensions/{id} when a +// Extension cards omit the README, release history, and requirements map. Consumers can fetch those fields from GET /extensions/{id} when a // visitor opens an extension's detail page. const ExtensionCardContentSchema = ExtensionContentSchema.omit({ readme: true, - releases: true + releases: true, + requires: true }).extend({ - website: httpUrl() + name: z.string().min(1).max(120), + description: z.string().min(1).max(4000), + version: z.string().min(1).max(100), + license: z.string().min(1).max(1000), + icon_url: z.string().max(2048).optional(), + homepage_url: httpUrl() .nullable() + .optional() .describe("Null when a legacy URL exceeds the card length bound"), download_url: httpUrl() .nullable() @@ -183,13 +185,13 @@ export const ExtensionListItemSchema = ExtensionCardContentSchema.extend({ export type ExtensionListItem = z.infer; // The published projection as its *owner* sees it. Identical to the -// catalogue's, except releases may be empty: v1 constrained +// public listing's, except releases may be empty: v1 constrained // extensions.releases to NOT NULL and nothing more, so a row adopted by // migration 0021 can legitimately have none, and the owner view is where // someone looks at what is actually there rather than at an idealised copy. // This can only ever describe a pre-v2 row - approve() requires a release -// before anything reaches the catalogue through v2. -const PublishedExtensionContentSchema = ExtensionContentSchema.extend({ +// before anything reaches the extension directory through v2. +const PublishedExtensionContentSchema = PublishedContentSchema.extend({ releases: z.array(StoredReleaseSchema).max(100) }); @@ -212,9 +214,9 @@ const PendingRevisionRefSchema = z .openapi("PendingRevisionRef"); // Set once a moderator pulls an already-published extension from the -// catalogue for cause (its upstream source disappearing, for example). +// extension directory for cause (its upstream source disappearing, for example). // Content and history are kept, so the owner can still see and edit the -// extension - they just cannot get it back into the catalogue without a +// extension - they just cannot get it back into the extension directory without a // moderator re-listing it. export const DelistedInfoSchema = z .object({ @@ -272,9 +274,9 @@ export const ExtensionListQuerySchema = z.object({ }); // Unified list query for the merged GET /extensions (optional auth, -// role-aware). scope selects the projection: public (published catalogue +// role-aware). scope selects the projection: public (published extension // cards, anonymous allowed), mine (caller's own Owned rows), all (moderator's -// whole-catalogue Owned rows). developer_id is public-only, status/q are +// view of all Owned rows). developer_id is public-only, status/q are // all-only; mismatched combinations are rejected with 422 so a caller cannot // silently get a different projection than requested. export const ExtensionListScopeSchema = z.enum(["public", "mine", "all"]); @@ -289,7 +291,7 @@ export const UnifiedExtensionListQuerySchema = ExtensionListQuerySchema.extend({ scope: ExtensionListScopeSchema.default("public").openapi({ param: { name: "scope", in: "query" }, description: - "public: published catalogue (anonymous allowed). mine: caller's own extensions. all: every extension (moderator only)." + "public: published extension directory (anonymous allowed). mine: caller's own extensions. all: every extension (moderator only)." }), developer_id: z .string() diff --git a/src/services/extensions/v2/schemas/manifest.ts b/src/services/extensions/v2/schemas/manifest.ts new file mode 100644 index 00000000..d1a8f9af --- /dev/null +++ b/src/services/extensions/v2/schemas/manifest.ts @@ -0,0 +1,73 @@ +import { z } from "@hono/zod-openapi"; +import { httpUrl } from "./common"; + +// Package metadata follows FOSSBilling/modules manifest.schema.json (v1). +// Package paths are kept relative; serving an icon is a publisher concern. +export const MANIFEST_EXTENSION_TYPES = [ + "module", + "theme", + "payment-gateway", + "server-manager", + "domain-registrar", + "translation" +] as const; + +export const ManifestRequirementsSchema = z + .object({ + fossbilling: z.string().min(1).max(1000), + php: z.string().min(1).max(1000), + modules: z + .record(z.string().regex(/^[a-z0-9_-]+$/), z.string().min(1).max(1000)) + .optional() + }) + .strict() + .openapi("ManifestRequirements"); + +export const ManifestSchema = z + .object({ + manifest_version: z.literal(1), + type: z.enum(MANIFEST_EXTENSION_TYPES), + id: z + .string() + .min(1) + .max(64) + .regex(/^[a-z0-9_-]+$/), + name: z.string().min(1).max(100), + version: z + .string() + .max(100) + .regex( + /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$/ + ), + description: z.string().min(1).max(500), + author: z + .object({ name: z.string().min(1).max(200), url: httpUrl().optional() }) + .strict(), + license: z + .string() + .min(1) + .max(1000) + .regex(/^[A-Za-z0-9.\-+_:() |]+$/), + homepage_url: httpUrl().optional(), + icon_url: z + .string() + .min(1) + .max(2048) + .regex(/^(?!\/)(?![a-zA-Z][a-zA-Z0-9+.-]*:)(?!.*\.\.)[^\\]+$/) + .optional(), + requires: ManifestRequirementsSchema + }) + .strict() + .openapi("Manifest"); + +export type Manifest = z.infer; + +export function manifestMetadata( + content: Pick +) { + return { + manifest_version: content.manifest_version, + author: content.author, + requires: content.requires + }; +} diff --git a/test/services/extensions/v1/index.test.ts b/test/services/extensions/v1/index.test.ts index bbc236c4..22b0bacf 100644 --- a/test/services/extensions/v1/index.test.ts +++ b/test/services/extensions/v1/index.test.ts @@ -125,7 +125,7 @@ describe("Extensions API v1", () => { // The assembled body is cached in CACHE_KV (60s TTL): the second request // must be served byte-identical from KV even after the underlying rows - // change, until revalidateCatalogue deletes the key or the TTL lapses. + // change, until revalidateExtensions deletes the key or the TTL lapses. it("should serve a repeated request from the cached body", async () => { const requestOnce = async () => { const ctx = createExecutionContext(); @@ -168,7 +168,7 @@ describe("Extensions API v1", () => { }); // Opt-in pagination walks the list in (LOWER(id), id) order - the same - // ordering as the catalogue index - so pages are deterministic and + // ordering as the extension directory index - so pages are deterministic and // case-mixed ids sort case-insensitively. Without params the response // stays the unpaginated legacy shape (no pagination object). it("should page deterministically with an opt-in limit and offset", async () => { @@ -274,7 +274,7 @@ describe("Extensions API v1", () => { expect(example!.releases[0].tag).toBe("0.0.5"); }); - // Legacy FOSSBilling installs (0.8.x) render the catalogue with + // Legacy FOSSBilling installs (0.8.x) render the extension directory with // strict_variables, where a missing key throws and crashes the whole // Extensions page (issue FOSSBilling/FOSSBilling#4486). The keys must // always be present ("" when unknown) so old templates fall into their diff --git a/test/services/extensions/v2/db-fixtures.ts b/test/services/extensions/v2/db-fixtures.ts index 76fbf9d0..8cc0db9d 100644 --- a/test/services/extensions/v2/db-fixtures.ts +++ b/test/services/extensions/v2/db-fixtures.ts @@ -274,7 +274,7 @@ export async function insertExtension( row.developer_id, row.published_at !== undefined ? row.published_at : now, row.published_revision_id ?? null, - row.type ?? "mod", + row.type ?? "module", row.name ?? "Extension", row.description ?? "d", row.releases ?? "[]", diff --git a/test/services/extensions/v2/extension-writes.test.ts b/test/services/extensions/v2/extension-writes.test.ts index 8d5f75e5..73103c82 100644 --- a/test/services/extensions/v2/extension-writes.test.ts +++ b/test/services/extensions/v2/extension-writes.test.ts @@ -83,7 +83,7 @@ describe("Extensions API v2 writes", () => { expect(data.error.code).toBe("VALIDATION_ERROR"); }); - it("rejects a spdx_id that isn't a current SPDX license identifier", async () => { + it("rejects the legacy license object", async () => { await seedDeveloper("new-developer", "user-1"); const res = await post( "/extensions/v2/extensions", @@ -97,42 +97,38 @@ describe("Extensions API v2 writes", () => { expect(await countRevisions(db)).toBe(0); }); - it("rejects a deprecated SPDX identifier", async () => { + it("rejects characters outside the manifest license grammar", async () => { await seedDeveloper("new-developer", "user-1"); const res = await post( "/extensions/v2/extensions", await authHeaders("user-1"), - // "GPL-3.0" is deprecated in favor of GPL-3.0-only / GPL-3.0-or-later. - { ...sampleCreate(), license: { name: "GPL 3.0", spdx_id: "GPL-3.0" } } + { ...sampleCreate(), license: "invalid/license" } ); expect(res.status).toBe(422); }); - it("stores a recognized spdx_id alongside the display name", async () => { + it("stores a manifest license expression", async () => { await seedDeveloper("new-developer", "user-1"); const res = await post( "/extensions/v2/extensions", await authHeaders("user-1"), { ...sampleCreate(), - license: { name: "Apache License 2.0", spdx_id: "Apache-2.0" } + license: "Apache-2.0 OR MIT" } ); expect(res.status).toBe(201); const data = (await res.json()) as { result: { revision_id: string } }; const revision = await getRevision(db, data.result.revision_id); - expect(JSON.parse(revision!.content).license).toEqual({ - name: "Apache License 2.0", - spdx_id: "Apache-2.0" - }); + expect(JSON.parse(revision!.content).license).toBe("Apache-2.0 OR MIT"); }); - it("accepts a license with no spdx_id, for custom/proprietary licenses", async () => { + it("accepts a custom manifest license", async () => { await seedDeveloper("new-developer", "user-1"); const res = await post( "/extensions/v2/extensions", await authHeaders("user-1"), - { ...sampleCreate(), license: { name: "Acme Proprietary License" } } + { ...sampleCreate(), license: "LicenseRef-Acme" } ); expect(res.status).toBe(201); }); @@ -295,7 +291,7 @@ describe("Extensions API v2 writes", () => { date: "2026-01-01T00:00:00Z", download_url: longUrl, changelog_url: longUrl, - min_fossbilling_version: "0.6" + requires: { fossbilling: ">=0.6", php: "*" } })); const body = { ...sampleCreate(), releases }; expect( @@ -329,12 +325,12 @@ describe("Extensions API v2 writes", () => { expect(await countRevisions(db)).toBe(0); }); - it("preserves compatibility with stored slug ids over 100 characters", async () => { + it("rejects new ids above the manifest limit of 64 characters", async () => { await seedDeveloper("d".repeat(120), "user-1"); const res = await createExtension("user-1", { - extensionId: "e".repeat(120) + extensionId: "e".repeat(65) }); - expect(res.status).toBe(201); + expect(res.status).toBe(422); }); it("caps each user's unreviewed backlog", async () => { @@ -774,7 +770,7 @@ describe("Extensions API v2 writes", () => { // Both of these are states the README's mapping table now documents, and // both were reachable before it did. - it("reports a live extension adopted from the pre-v2 catalogue", async () => { + it("reports a live extension adopted from the pre-v2 directory", async () => { await seedOwnedExtension(); const res = await get( @@ -876,7 +872,7 @@ describe("Extensions API v2 writes", () => { ); const res = await get( - "/extensions/v2/extensions?scope=mine&type=mod", + "/extensions/v2/extensions?scope=mine&type=module", headers ); expect(res.status).toBe(200); diff --git a/test/services/extensions/v2/harness.ts b/test/services/extensions/v2/harness.ts index fe9967cf..a66e0012 100644 --- a/test/services/extensions/v2/harness.ts +++ b/test/services/extensions/v2/harness.ts @@ -129,7 +129,10 @@ export async function syncIdentity(sub: string, body: unknown) { // are now properties of the extension record rather than of the edit. export function sampleContent(overrides?: { name?: string }) { return { - type: "mod", + manifest_version: 1, + author: { name: "Developer" }, + requires: { fossbilling: ">=0.6", php: ">=8.2" }, + type: "module", name: overrides?.name ?? "New Extension", description: "A new extension", releases: [ @@ -137,11 +140,11 @@ export function sampleContent(overrides?: { name?: string }) { tag: "1.0.0", date: "2026-01-01T00:00:00Z", download_url: "https://example.com/download.zip", - min_fossbilling_version: "0.6" + requires: { fossbilling: ">=0.6", php: ">=8.2" } } ], - website: "https://example.com", - license: { name: "MIT" }, + homepage_url: "https://example.com", + license: "MIT", readme: "# Readme", source: { type: "github", repo: "example/new-ext" }, version: "1.0.0", diff --git a/test/services/extensions/v2/manifest.test.ts b/test/services/extensions/v2/manifest.test.ts new file mode 100644 index 00000000..00d82dc6 --- /dev/null +++ b/test/services/extensions/v2/manifest.test.ts @@ -0,0 +1,148 @@ +import { describe, expect, it, vi } from "vitest"; +import { + setupExtensionsV2Tests, + seedDeveloper, + authHeaders, + post, + put, + get, + sampleCreate, + sampleContent, + db +} from "./harness"; +import { insertUser } from "./db-fixtures"; + +vi.mock("@octokit/request", async () => + (await import("../../../mocks/octokit")).octokitRequestMock() +); +setupExtensionsV2Tests(); + +describe("manifest v1 publishing", () => { + it("publishes package metadata unchanged and keeps ownership distinct from attribution", async () => { + await seedDeveloper("owner", "user-1"); + await insertUser(db, { id: "moderator", is_moderator: 1 }); + const content = { + ...sampleCreate({ extensionId: "my_module" }), + author: { name: "Package Author", url: "https://author.example.com" }, + icon_url: "assets/icon.svg", + license: "MIT OR Apache-2.0", + requires: { + fossbilling: "^0.8.9 || ^1.0", + php: ">=8.3 <9", + modules: { branding: "^1.2" } + } + }; + const created = await post( + "/extensions/v2/extensions", + await authHeaders("user-1"), + content + ); + expect(created.status).toBe(201); + const { result } = (await created.json()) as { + result: { revision_id: string }; + }; + expect( + ( + await post( + `/extensions/v2/extensions/my_module/revisions/${result.revision_id}/approve`, + await authHeaders("moderator"), + {} + ) + ).status + ).toBe(200); + const detail = await get("/extensions/v2/extensions/my_module", {}); + const { result: extension } = (await detail.json()) as { + result: Record; + }; + expect(extension).toMatchObject({ ...content, developer: { id: "owner" } }); + expect(extension).not.toHaveProperty("website"); + const list = await get("/extensions/v2/extensions?type=module", {}); + const { result: items } = (await list.json()) as { + result: Record[]; + }; + expect(items[0]).not.toHaveProperty("requires"); + const updated = { + ...sampleContent(), + author: content.author, + requires: { fossbilling: ">=1.0.0", php: ">=8.4" } + }; + const edit = await put( + "/extensions/v2/extensions/my_module", + await authHeaders("user-1"), + updated + ); + expect(edit.status).toBe(202); + const { result: revision } = (await edit.json()) as { + result: { revision_id: string }; + }; + expect( + ( + await post( + `/extensions/v2/extensions/my_module/revisions/${revision.revision_id}/approve`, + await authHeaders("moderator"), + {} + ) + ).status + ).toBe(200); + await expect( + (await get("/extensions/v2/extensions/my_module", {})).json() + ).resolves.toMatchObject({ + result: { requires: updated.requires, license: "MIT" } + }); + const legacyList = await get("/extensions/v1/list?type=mod", {}); + await expect(legacyList.json()).resolves.toMatchObject({ + result: [{ id: "my_module", type: "mod" }] + }); + const legacy = await get("/extensions/v1/my_module", {}); + expect(legacy.status).toBe(200); + await expect(legacy.json()).resolves.toMatchObject({ + result: { + type: "mod", + website: "https://example.com", + license: { name: "MIT" } + } + }); + }); + + it("accepts an omitted homepage and rejects old fields and unsafe package paths", async () => { + await seedDeveloper("owner", "user-1"); + const headers = await authHeaders("user-1"); + const { homepage_url: _homepage, ...content } = sampleCreate(); + for (const invalid of [ + { type: "mod" }, + { manifest_version: 2 }, + { license: { name: "MIT" } }, + { website: "https://example.com" }, + { version: "v1.0.0" }, + { version: "01.0.0" }, + { version: "1.0.0+" + "x".repeat(100) }, + { description: "x".repeat(501) }, + { name: "x".repeat(101) }, + { requires: { fossbilling: "^1" } }, + { + requires: { + fossbilling: "^1", + php: "^8", + modules: { "bad/dependency": "*" } + } + }, + ...[ + "../icon.svg", + "/icon.svg", + "https://example.com/icon.svg", + "a\\icon.svg" + ].map((icon_url) => ({ icon_url })) + ]) + expect( + ( + await post("/extensions/v2/extensions", headers, { + ...content, + ...invalid + }) + ).status + ).toBe(422); + expect( + (await post("/extensions/v2/extensions", headers, content)).status + ).toBe(201); + }); +}); diff --git a/test/services/extensions/v2/migrations.test.ts b/test/services/extensions/v2/migrations.test.ts index 63f2e49d..38b309af 100644 --- a/test/services/extensions/v2/migrations.test.ts +++ b/test/services/extensions/v2/migrations.test.ts @@ -259,7 +259,7 @@ describe("Extensions D1 migrations", () => { } }); - // idx_extensions_id_nocase cannot be created over a catalogue that already + // idx_extensions_id_nocase cannot be created over a directory that already // holds a case-colliding pair. Without a check first, the failure lands // mid-rebuild as a bare UNIQUE constraint error naming no rows. it("0021 refuses to run against ids that differ only in case", () => { @@ -683,7 +683,7 @@ describe("Extensions D1 migrations", () => { db.exec(migration("0021_restructure_extensions_revisions.sql")); - // Rows that were already in the catalogue are published; a submission + // Rows that were already in the extension directory are published; a submission // that only ever proposed an id becomes an unpublished extension. expect( db @@ -972,3 +972,126 @@ describe("Resource-bound migration", () => { } }); }); + +it("migrates published manifest metadata without changing reviewed audit bodies", () => { + const db = new DatabaseSync(":memory:"); + try { + db.exec("PRAGMA foreign_keys=ON"); + for (const name of migrationNames.filter((name) => name < "0029")) + db.exec(migration(name)); + db.exec( + "INSERT INTO users(id,created_at,updated_at) VALUES('owner','now','now')" + ); + db.exec( + "INSERT INTO developers(id,type,name,url,owner_user_id) VALUES('dev','organization','Developer','https://example.com','owner')" + ); + const releases = JSON.stringify([ + { + tag: "1.0.0", + date: "2026-01-01", + download_url: "https://example.com/a.zip", + min_fossbilling_version: "0.8.9" + } + ]); + db.prepare( + `INSERT INTO extensions(id,developer_id,type,name,description,releases,website,license,readme,source,version,download_url,published_at) + VALUES('legacy','dev','mod','Legacy','Description',?,'https://example.com',?,'README','{}','1.0.0','https://example.com/a.zip','now')` + ).run( + releases, + JSON.stringify({ + name: "Apache License", + spdx_id: "Apache-2.0", + URL: "https://example.com/license" + }) + ); + const audit = JSON.stringify({ + type: "mod", + website: "https://example.com", + license: { name: "MIT" } + }); + db.prepare( + "INSERT INTO extension_revisions(id,extension_id,developer_id,submitted_by,status,content) VALUES('audit','legacy','dev','owner','approved',?)" + ).run(audit); + // Existing over-quota collections must survive a structural backfill. + db.exec( + "UPDATE extension_resource_usage SET extensions=101 WHERE scope='developer'" + ); + db.exec("BEGIN"); + db.exec(migration("0029_align_manifest.sql")); + db.exec("COMMIT"); + const row = db + .prepare( + "SELECT type,license,manifest,published_bytes FROM extensions WHERE id='legacy'" + ) + .get()!; + expect(() => + db + .prepare( + "UPDATE extensions SET manifest=manifest || ' ' WHERE id='legacy'" + ) + .run() + ).toThrow(/extension_resource_quota/); + db.exec( + "UPDATE extension_resource_usage SET extensions=1 WHERE scope='developer'" + ); + expect( + JSON.parse( + db.prepare("SELECT releases FROM extensions WHERE id='legacy'").get() + ?.releases as string + ) + ).toEqual([ + { + tag: "1.0.0", + date: "2026-01-01", + download_url: "https://example.com/a.zip", + requires: { fossbilling: ">=0.8.9", php: "*" } + } + ]); + expect(row.type).toBe("module"); + expect(JSON.parse(row.license as string)).toBe("Apache-2.0"); + expect(JSON.parse(row.manifest as string)).toEqual({ + manifest_version: 1, + author: { name: "Developer", url: "https://example.com" }, + requires: { fossbilling: ">=0.8.9", php: "*" } + }); + expect( + db + .prepare("SELECT content FROM extension_revisions WHERE id='audit'") + .get()?.content + ).toBe(audit); + expect( + db + .prepare( + readFileSync( + join(migrationsDirectory, "../resource-inventory.sql"), + "utf8" + ) + ) + .all() + ).toEqual([]); + db.prepare("UPDATE extensions SET manifest=? WHERE id='legacy'").run( + JSON.stringify({ + manifest_version: 1, + author: { name: "Changed" }, + requires: { + fossbilling: "^0.8.9", + php: ">=8.3", + modules: { branding: "^1" } + } + }) + ); + expect( + db + .prepare( + readFileSync( + join(migrationsDirectory, "../resource-inventory.sql"), + "utf8" + ) + ) + .all() + ).toEqual([]); + expect(db.prepare("PRAGMA foreign_key_check").all()).toEqual([]); + } finally { + db.close(); + } +}); diff --git a/test/services/extensions/v2/moderation-revalidate.test.ts b/test/services/extensions/v2/moderation-revalidate.test.ts index 51a12bc1..cef63803 100644 --- a/test/services/extensions/v2/moderation-revalidate.test.ts +++ b/test/services/extensions/v2/moderation-revalidate.test.ts @@ -81,7 +81,7 @@ async function delist(as: string): Promise { ); } -describe("CDN cache revalidation on catalogue mutations", () => { +describe("CDN cache revalidation on extension mutations", () => { it("does not revalidate unchanged profile replays", async () => { const fetcher = stubFrontend(); const headers = await authHeaders("noop-revalidate-owner"); @@ -105,7 +105,7 @@ describe("CDN cache revalidation on catalogue mutations", () => { expect(fetcher.fetch).toHaveBeenCalledTimes(2); }); - it("purges the catalogue tags after a successful delist", async () => { + it("purges the extension page tags after a successful delist", async () => { await seedModAndExtension(); const fetcher = stubFrontend(); @@ -123,11 +123,11 @@ describe("CDN cache revalidation on catalogue mutations", () => { "content-type": "application/json" }); expect(JSON.parse(String(init.body))).toEqual({ - tags: ["catalogue", "developers"] + tags: ["extensions", "developers"] }); }); - it("purges the catalogue tags after a successful relist", async () => { + it("purges the extension page tags after a successful relist", async () => { await seedModAndExtension(); expect((await delist("mod-1")).status).toBe(200); const fetcher = stubFrontend(); @@ -146,7 +146,7 @@ describe("CDN cache revalidation on catalogue mutations", () => { ); expect(init.method).toBe("POST"); expect(JSON.parse(String(init.body))).toEqual({ - tags: ["catalogue", "developers"] + tags: ["extensions", "developers"] }); }); diff --git a/test/services/extensions/v2/moderation.test.ts b/test/services/extensions/v2/moderation.test.ts index d2a2a324..e9f0a303 100644 --- a/test/services/extensions/v2/moderation.test.ts +++ b/test/services/extensions/v2/moderation.test.ts @@ -189,7 +189,7 @@ describe("Extensions API v2", () => { expect(res.status).toBe(403); }); - it("lists extensions across every status, unlike the public catalogue", async () => { + it("lists extensions across every status, unlike the public extension listing", async () => { await insertUser(db, { id: "mod-1", is_moderator: 1 }); await seedDeveloper("new-developer", "user-1"); await insertExtension(db, { @@ -405,7 +405,7 @@ describe("Extensions API v2", () => { await seedDeveloper("new-developer", "user-1"); const { id, revisionId } = await createPending("user-1"); - // Until approval the extension exists but is in neither catalogue. + // Until approval the extension exists but is in neither public listing. expect((await get("/extensions/v1/new-ext", {})).status).toBe(404); expect((await get("/extensions/v2/extensions/new-ext", {})).status).toBe( 404 @@ -532,7 +532,7 @@ describe("Extensions API v2", () => { // Revision history outlives the rules its content was written under, so // the response schema tolerates a stored revision with no releases. The - // public catalogue does not, which makes approval the boundary that has to + // public extension listing does not, which makes approval the boundary that has to // re-check rather than trust submission-time validation. it("refuses to publish a revision that predates current content rules", async () => { await insertUser(db, { id: "mod-1", is_moderator: 1 }); @@ -911,7 +911,7 @@ describe("Extensions API v2", () => { expect(res.status).toBe(422); }); - it("removes a published extension from the public catalogue and records why", async () => { + it("removes a published extension from the public extension listing and records why", async () => { await insertUser(db, { id: "mod-1", is_moderator: 1 }); await seedDeveloper("new-developer", "user-1"); await insertExtension(db, { @@ -982,7 +982,7 @@ describe("Extensions API v2", () => { // Public reads are covered in public-extensions.test.ts. This is about // who can still reach a delisted extension's full record once it is out - // of the catalogue: the owner and moderators via the merged + // of the extension directory: the owner and moderators via the merged // GET /extensions/{id} (ownership/moderator check, unaffected by delisted // state - see getOwned()), and no one else (404 hides existence). it("only the owner or a moderator can still reach a delisted extension", async () => { @@ -1108,7 +1108,7 @@ describe("Extensions API v2", () => { }); }); - it("restores a delisted extension to the public catalogue", async () => { + it("restores a delisted extension to the public extension listing", async () => { await insertUser(db, { id: "mod-1", is_moderator: 1 }); await seedDeveloper("new-developer", "user-1"); await insertExtension(db, { diff --git a/test/services/extensions/v2/public-extensions.test.ts b/test/services/extensions/v2/public-extensions.test.ts index d5b064fd..449019eb 100644 --- a/test/services/extensions/v2/public-extensions.test.ts +++ b/test/services/extensions/v2/public-extensions.test.ts @@ -12,19 +12,19 @@ setupExtensionsV2Tests(); describe("Extensions API v2", () => { describe("GET /extensions", () => { - async function seedCatalogue(ids: string[]): Promise { + async function seedExtensions(ids: string[]): Promise { await insertDeveloper(db, { - id: "catalogue-developer", + id: "listing-developer", type: "user", - name: "Catalogue Developer", + name: "Listing Developer", url: null, owner_user_id: null }); for (const id of ids) { await insertExtension(db, { id, - type: "mod", - developer_id: "catalogue-developer", + type: "module", + developer_id: "listing-developer", name: id, description: `Description for ${id}`, releases: '[{"tag":"1.0.0"}]', @@ -32,7 +32,7 @@ describe("Extensions API v2", () => { license: '{"name":"MIT"}', icon_url: null, readme: `README for ${id}`, - source: '{"type":"github","repo":"example/catalogue"}', + source: '{"type":"github","repo":"example/extensions"}', version: "1.0.0", download_url: "https://example.com/download.zip" }); @@ -61,7 +61,7 @@ describe("Extensions API v2", () => { }); it("marks unowned public developers as unclaimed", async () => { - await seedCatalogue(["legacy-extension"]); + await seedExtensions(["legacy-extension"]); const res = await get("/extensions/v2/extensions", {}); expect(res.status).toBe(200); @@ -74,7 +74,7 @@ describe("Extensions API v2", () => { it("filters by type", async () => { await seedOwnedExtension(); - const matching = await get("/extensions/v2/extensions?type=mod", {}); + const matching = await get("/extensions/v2/extensions?type=module", {}); const matchingBody = (await matching.json()) as { result: unknown[] }; expect(matchingBody.result).toHaveLength(1); @@ -111,7 +111,7 @@ describe("Extensions API v2", () => { }); it("returns deterministic first, middle, and final pages", async () => { - await seedCatalogue(["charlie", "Alpha", "bravo", "delta", "echo"]); + await seedExtensions(["charlie", "Alpha", "bravo", "delta", "echo"]); const first = await get("/extensions/v2/extensions?limit=2", {}); const firstBody = (await first.json()) as { @@ -159,7 +159,7 @@ describe("Extensions API v2", () => { }); it("supports UTF-8 extension ids in cursors", async () => { - await seedCatalogue(["alpha", "zulu", "éclair", "😀"]); + await seedExtensions(["alpha", "zulu", "éclair", "😀"]); const first = await get("/extensions/v2/extensions?limit=3", {}); const firstBody = (await first.json()) as { @@ -180,7 +180,7 @@ describe("Extensions API v2", () => { }); it("accepts the maximum limit and rejects values above it", async () => { - await seedCatalogue(["one"]); + await seedExtensions(["one"]); expect( (await get("/extensions/v2/extensions?limit=100", {})).status ).toBe(200); @@ -224,15 +224,15 @@ describe("Extensions API v2", () => { it("404s for a delisted extension, even though it is still published", async () => { await insertDeveloper(db, { - id: "catalogue-developer", + id: "listing-developer", type: "user", - name: "Catalogue Developer", + name: "Listing Developer", url: null, owner_user_id: null }); await insertExtension(db, { id: "delisted-ext", - developer_id: "catalogue-developer", + developer_id: "listing-developer", delisted_at: "2026-01-01T00:00:00.000Z", delist_reason: "Upstream source removed" }); diff --git a/test/services/extensions/v2/resource-bounds.test.ts b/test/services/extensions/v2/resource-bounds.test.ts index 3d28fcb9..8f37b28e 100644 --- a/test/services/extensions/v2/resource-bounds.test.ts +++ b/test/services/extensions/v2/resource-bounds.test.ts @@ -191,7 +191,8 @@ describe("Extension resource admission", () => { "readme", "source", "version", - "download_url" + "download_url", + "manifest" ]; async function reconcile() { const rows = await db @@ -245,7 +246,12 @@ describe("Extension resource admission", () => { "読んでください", JSON.stringify({ type: "custom", repo: "café/😀" }), "1.0.0", - "https://example.test/é.zip" + "https://example.test/é.zip", + JSON.stringify({ + manifest_version: 1, + author: { name: "Déveloper" }, + requires: { fossbilling: "*", php: "*" } + }) ]; for (let i = 0; i < columns.length; i++) { await db @@ -928,7 +934,7 @@ describe("Bounded revision reads and maintenance", () => { await insertUser(db, { id: "mod", is_moderator: 1 }); const content = { ...sampleContent(), - description: "\u0001".repeat(4000), + description: "\u0001".repeat(500), readme: "" }; const remaining = @@ -1120,18 +1126,21 @@ describe("Bounded revision reads and maintenance", () => { ExtensionListItemSchema.safeParse(cards.data?.items[0]).success ).toBe(true); expect(cards.data?.items[0]).toMatchObject({ - website: null, - license: { name: "L".repeat(100) }, + license: "L".repeat(1000), source: { type: "custom", repo: "R".repeat(500) } }); const publicDetail = await extensions.getById("live"); expect(publicDetail.error).toBeNull(); - expect(publicDetail.data).toMatchObject({ website, license, source }); + expect(publicDetail.data).toMatchObject({ + homepage_url: website, + license: license.name, + source + }); const ownerDetail = await extensions.getOwned("live"); expect(ownerDetail.error).toBeNull(); expect(ownerDetail.data?.extension.published).toMatchObject({ - website, - license, + homepage_url: website, + license: license.name, source }); }); diff --git a/wrangler.jsonc b/wrangler.jsonc index 639225d5..ac30eea0 100644 --- a/wrangler.jsonc +++ b/wrangler.jsonc @@ -69,7 +69,7 @@ "services": [ { // Private service binding to the extensions site worker. Used by - // revalidateCatalogue() to purge the site's CDN-cached catalogue pages + // revalidateExtensions() to purge the site's CDN-cached extension pages // after content mutations — see src/services/extensions/v2/revalidate.ts. "binding": "EXTENSIONS_FRONTEND", "service": "extensions" From 9dc8abfae76b87061e1f8f501d42d3eba939110f Mon Sep 17 00:00:00 2001 From: Adam Daley Date: Wed, 4 Nov 2026 03:10:15 +0000 Subject: [PATCH 2/5] Require canonical manifest data after migration --- src/services/extensions/v1/database.ts | 92 +++++++------ src/services/extensions/v2/README.md | 17 ++- src/services/extensions/v2/db/extensions.ts | 53 ++------ .../extensions/v2/db/manifest-preflight.sql | 46 +++++++ .../v2/db/migrations/0029_align_manifest.sql | 82 +++++++++--- test/services/extensions/v1/index.test.ts | 81 ++++++++++-- test/services/extensions/v2/account.test.ts | 4 +- test/services/extensions/v2/db-fixtures.ts | 15 ++- .../services/extensions/v2/migrations.test.ts | 123 +++++++++++++++++- .../extensions/v2/public-extensions.test.ts | 2 +- .../extensions/v2/resource-bounds.test.ts | 8 +- 11 files changed, 398 insertions(+), 125 deletions(-) create mode 100644 src/services/extensions/v2/db/manifest-preflight.sql diff --git a/src/services/extensions/v1/database.ts b/src/services/extensions/v1/database.ts index e37cb434..f127b745 100644 --- a/src/services/extensions/v1/database.ts +++ b/src/services/extensions/v1/database.ts @@ -1,10 +1,11 @@ -import { minVersion } from "semver"; -import { and, eq, isNotNull, isNull, or, sql } from "drizzle-orm"; +import { minVersion, validRange } from "semver"; +import { and, eq, isNotNull, isNull, sql } from "drizzle-orm"; import { ExtensionsDb } from "../../../lib/db"; import { extensions, developers } from "../v2/db/schema"; import { DatabaseResult } from "../../../lib/interfaces"; import { sortReleasesDescending } from "../../../lib/releases"; import { parseJSON } from "../../../lib/json"; +import type { Release as ManifestRelease } from "../v2/schemas/extensions"; import { Extension, Release, Author, Repository } from "./interfaces"; const EXTENSION_COLUMNS = { @@ -64,6 +65,10 @@ export class ExtensionsDatabase { // Shared catch shape for this module's D1 reads: driver message surfaced // under the generic DATABASE_ERROR code. private databaseError(error: unknown): DatabaseResult { + console.error( + "[extensions-v1] Cannot read canonical extension data:", + error + ); return { data: null, error: { @@ -106,12 +111,7 @@ export class ExtensionsDatabase { type ? and( published, - type === "mod" - ? or( - eq(extensions.type, "mod"), - eq(extensions.type, "module") - ) - : eq(extensions.type, type) + eq(extensions.type, type === "mod" ? "module" : type) ) : published ); @@ -131,10 +131,14 @@ export class ExtensionsDatabase { const hasMore = page ? rows.length > page.limit : false; const trimmed = page && hasMore ? rows.slice(0, page.limit) : rows; - return { - data: { extensions: trimmed.map(parseExtensionRow), hasMore }, - error: null - }; + try { + return { + data: { extensions: trimmed.map(parseExtensionRow), hasMore }, + error: null + }; + } catch (error) { + return this.databaseError(error); + } } async getExtensionById(id: string): Promise> { @@ -160,7 +164,11 @@ export class ExtensionsDatabase { return this.notFound(id); } - return { data: parseExtensionRow(row), error: null }; + try { + return { data: parseExtensionRow(row), error: null }; + } catch (error) { + return this.databaseError(error); + } } // Badge/version endpoints read only these two columns, and badges are the @@ -193,19 +201,23 @@ export class ExtensionsDatabase { return this.notFound(id); } - const releases = sortReleasesDescending(legacyReleases(row.releases)); - return { - data: { - latestRelease: releases[0] ?? null, - license: legacyLicense(row.license) - }, - error: null - }; + try { + const releases = sortReleasesDescending(toV1Releases(row.releases)); + return { + data: { + latestRelease: releases[0] ?? null, + license: toV1License(row.license) + }, + error: null + }; + } catch (error) { + return this.databaseError(error); + } } } function parseExtensionRow(row: ExtensionRow): Extension { - const releases = legacyReleases(row.releases); + const releases = toV1Releases(row.releases); return { id: row.id, type: (row.type === "module" ? "mod" : row.type) as Extension["type"], @@ -223,7 +235,7 @@ function parseExtensionRow(row: ExtensionRow): Extension { } as Author, releases: sortReleasesDescending(releases), website: row.website, - license: legacyLicense(row.license), + license: toV1License(row.license), // Package-relative manifest icons have no public URL in the v1 contract. icon_url: row.iconUrl && /^https?:\/\//i.test(row.iconUrl) ? row.iconUrl : "", @@ -234,26 +246,22 @@ function parseExtensionRow(row: ExtensionRow): Extension { }; } -function legacyLicense(stored: string): { name: string; URL?: string } { - const value = parseJSON(stored, { - name: "" - }); - return typeof value === "string" ? { name: value } : value; +// Storage is canonical after migration 0029; these only adapt the v1 contract. +function toV1License(stored: string): { name: string } { + return { name: JSON.parse(stored) as string }; } -function legacyReleases(stored: string): Release[] { - return parseJSON>( - stored, - [] - ).map((release) => { - const { requires, ...legacy } = release; - if (legacy.min_fossbilling_version) return legacy; - let minimum = "0.0.0"; - try { - minimum = minVersion(requires?.fossbilling ?? "*")?.version ?? minimum; - } catch { - /* Composer-only ranges have no v1 representation. */ +function toV1Releases(stored: string): Release[] { + return (JSON.parse(stored) as ManifestRelease[]).map( + ({ requires, ...release }) => { + const range = validRange(requires.fossbilling); + const minimum = range === null ? null : minVersion(range); + if (!minimum) { + throw new Error( + `Release "${release.tag}" requires manual correction: FOSSBilling constraint "${requires.fossbilling}" has no representable v1 minimum` + ); + } + return { ...release, min_fossbilling_version: minimum.version }; } - return { ...legacy, min_fossbilling_version: minimum }; - }); + ); } diff --git a/src/services/extensions/v2/README.md b/src/services/extensions/v2/README.md index 524b2d70..ffdedb98 100644 --- a/src/services/extensions/v2/README.md +++ b/src/services/extensions/v2/README.md @@ -23,11 +23,24 @@ API publishing fields `readme`, `source`, `download_url`, and `releases` remain `author` is package attribution. `developer` identifies the authenticated owner and remains controlled by the API; an author field never grants ownership. -Migration `0029_align_manifest.sql` backfills published metadata from the owning developer and release requirements. The selected release's former minimum becomes `>=minimum`; missing requirements, including historical PHP requirements, become `*` to indicate that no constraint was recorded. License identifiers use the old `spdx_id` when available, otherwise the display name. Physical `website` storage remains for v1 compatibility. Reviewed and pending revision bodies retain their original audit content; old pending revisions must be resubmitted using the new contract before approval. Existing IDs and published historical content remain readable without being forced through new-write validation. +Run the read-only `db/manifest-preflight.sql` query before deployment. It reports extension IDs and zero-based release indexes needing manual correction. Every published release must have an explicitly reviewed `requires` object, including its PHP constraint; migration does not invent `*`, an author, or a license placeholder. Use the package manifest/release metadata to fill missing requirements. A deliberate `*` is allowed only when it accurately represents the reviewed requirement. + +Migration `0029_align_manifest.sql` stops with `manifest_data_requires_manual_correction` before changing library data if the preflight reports issues. Correct those rows and retry. Once prepared, it converts `mod` to `module`, converts legacy license objects to strings (using `spdx_id`, otherwise `name`), copies the current release's exact requirements and the developer's attribution, and removes obsolete release minimum fields. Physical `website` storage remains for v1 compatibility. Reviewed and pending revision bodies retain their original audit content; old pending revisions must be resubmitted using the new contract before approval. Published reads expect canonical storage; historical audit reads remain separate. + +For example, after checking an affected package's actual requirements, an operator can add `requires` to one release without rewriting its remaining fields: + +```sql +UPDATE extensions +SET releases = json_set(releases, '$[0].requires', + json('{"fossbilling":">=0.8.9","php":">=8.3"}')) +WHERE id = 'affected-extension'; +``` + +Use the ID, index, and verified constraints from the preflight investigation; the values above are examples. Re-run the query until it returns no rows. Apply the migration and API deployment in a controlled cutover: the old Worker expects object licenses, while the new Worker requires the added metadata column. -This changes the v2 request/response contract: regenerate the Extensions site's client and update CI publishers together with the API rollout. The modules publisher must stop translating `module` to `mod` and send the manifest fields directly. Package-relative icons use v1's empty-string fallback. The v1 adapter continues to return `mod`, `website`, object licenses, and minimum release versions; complex ranges are reduced to the earliest SemVer version when representable, with `0.0.0` for Composer-only ranges that v1 cannot express. +This changes the v2 request/response contract: regenerate the Extensions site's client and update CI publishers together with the API rollout. The modules publisher must stop translating `module` to `mod` and send the manifest fields directly. Package-relative icons use v1's empty-string fallback. The v1 adapter continues to return `mod`, `website`, object licenses, and minimum release versions; constraints are converted to their earliest matching SemVer version. A malformed, unsatisfiable, or Composer-only constraint with no representable v1 minimum produces an explicit error identifying the release for manual correction; it is never replaced with `0.0.0`. The frontend revalidation tags are `extensions` and `developers`, shared with the Extensions site. Deploy the API and site tag change together. New code calls `revalidateExtensions(c)`. diff --git a/src/services/extensions/v2/db/extensions.ts b/src/services/extensions/v2/db/extensions.ts index 11eb4c70..feadbb55 100644 --- a/src/services/extensions/v2/db/extensions.ts +++ b/src/services/extensions/v2/db/extensions.ts @@ -61,10 +61,7 @@ const CONTENT_COLUMNS = { >`CASE WHEN length(${extensions.website}) <= 2048 THEN ${extensions.website} ELSE NULL END`, license: sql< string | null - >`CASE WHEN length(CAST(${extensions.license} AS BLOB)) <= ${MAX_CONTENT_BYTES} AND json_valid(${extensions.license}) THEN - json_quote(substr(CASE WHEN json_type(${extensions.license}) = 'text' THEN json_extract(${extensions.license}, '$') - ELSE COALESCE(json_extract(${extensions.license}, '$.spdx_id'), json_extract(${extensions.license}, '$.name'), 'Unavailable') END, 1, 1000)) - ELSE '"Unavailable"' END`, + >`json_quote(substr(json_extract(${extensions.license}, '$'), 1, 1000))`, manifest: sql< string | null >`CASE WHEN length(CAST(${extensions.manifest} AS BLOB)) <= ${MAX_CONTENT_BYTES} AND json_valid(${extensions.manifest}) THEN @@ -1366,14 +1363,6 @@ function cardUrl(url: string | null): string | null { return url && url.length <= 2048 ? url : null; } -function parseLicense(stored: string | null): License { - const value = parseJSON( - stored ?? "", - "Unavailable" - ); - return typeof value === "string" ? value : (value.spdx_id ?? value.name); -} - function cardSource(stored: string | null): Repository { const source = parseJSON(stored ?? "", { type: "custom", @@ -1432,17 +1421,15 @@ function listContent( author: card ? { ...metadata.author, name: metadata.author.name.slice(0, 200) } : metadata.author, - type: (row.type === "mod" - ? "module" - : row.type) as ExtensionContent["type"], + type: row.type as ExtensionContent["type"], name: card ? (row.name as string).slice(0, 120) : (row.name as string), description: card ? (row.description as string).slice(0, 4000) : (row.description as string), homepage_url: (card ? cardUrl(row.website) : row.website) || undefined, license: card - ? parseLicense(row.license).slice(0, 1000) - : parseLicense(row.license), + ? (JSON.parse(row.license as string) as License).slice(0, 1000) + : (JSON.parse(row.license as string) as License), icon_url: (card ? cardUrl(row.iconUrl) : row.iconUrl) ?? undefined, source: card ? cardSource(row.source) @@ -1497,7 +1484,7 @@ function parseRow(row: PublishedRow): Extension { homepage_url: row.website || undefined, download_url: row.downloadUrl, readme: row.readme, - releases: boundedReleases(parseJSON(row.releases, [])) + releases: boundedReleases(JSON.parse(row.releases)) }; } @@ -1537,9 +1524,7 @@ function parseOwnedRow(row: OwnedRow): OwnedExtension { homepage_url: row.website || undefined, download_url: row.downloadUrl as string, readme: row.readme as string, - releases: boundedReleases( - parseJSON(row.releases as string, []) - ) + releases: boundedReleases(JSON.parse(row.releases as string)) } : null, pending_revision: @@ -1559,7 +1544,7 @@ export function parseContent(stored: string | null): StoredExtensionContent { const content = parseJSON(stored ?? "", {}); return { ...content, - releases: boundedReleases(content.releases ?? [], false) + releases: boundedReleases(content.releases ?? []) }; } @@ -1567,7 +1552,7 @@ export class LegacyContentError extends Error {} // Legacy bodies bypassed today's schema. Bound the collection and tag work // before semver sorting, while retaining support for partial historical content. -function boundedReleases(value: unknown, normalize = true): Release[] { +function boundedReleases(value: unknown): Release[] { if ( !Array.isArray(value) || value.length > 100 || @@ -1582,21 +1567,7 @@ function boundedReleases(value: unknown, normalize = true): Release[] { throw new LegacyContentError( "Legacy release collection exceeds safe read bounds" ); - if (!normalize) return sortReleasesDescending(value as Release[]); - return sortReleasesDescending( - value.map((release) => { - const { min_fossbilling_version, ...rest } = release; - return { - ...rest, - requires: rest.requires ?? { - fossbilling: min_fossbilling_version - ? `>=${min_fossbilling_version}` - : "*", - php: "*" - } - }; - }) as Release[] - ); + return sortReleasesDescending(value as Release[]); } export function oversizedContent(): DatabaseResult { @@ -1613,9 +1584,5 @@ export function oversizedContent(): DatabaseResult { function parseManifestMetadata( stored: string | null ): Pick { - return parseJSON(stored ?? "", { - manifest_version: 1, - author: { name: "Unknown" }, - requires: { fossbilling: "*", php: "*" } - }); + return JSON.parse(stored as string); } diff --git a/src/services/extensions/v2/db/manifest-preflight.sql b/src/services/extensions/v2/db/manifest-preflight.sql new file mode 100644 index 00000000..815e8df8 --- /dev/null +++ b/src/services/extensions/v2/db/manifest-preflight.sql @@ -0,0 +1,46 @@ +-- Read-only pre-deployment report. Correct every returned row before 0029. +-- Revision audit bodies are excluded. +WITH published AS ( + SELECT e.*, d.name AS author_name, d.url AS author_url, + CASE WHEN json_valid(e.license) THEN CASE json_type(e.license) + WHEN 'text' THEN json_extract(e.license,'$') + WHEN 'object' THEN CASE + WHEN json_type(e.license,'$.spdx_id') = 'text' THEN json_extract(e.license,'$.spdx_id') + WHEN json_type(e.license,'$.spdx_id') IS NULL OR json_type(e.license,'$.spdx_id') = 'null' + THEN CASE WHEN json_type(e.license,'$.name') = 'text' THEN json_extract(e.license,'$.name') END + END END END AS manifest_license + FROM extensions e JOIN developers d ON d.id = e.developer_id + WHERE e.published_at IS NOT NULL +), release_rows AS ( + SELECT p.id, r.key AS release_index, r.type AS release_type, + CASE WHEN r.type = 'object' THEN r.value ELSE '{}' END AS release + FROM published p, json_each(CASE WHEN json_valid(p.releases) THEN p.releases ELSE '[]' END) r +) +SELECT p.id, NULL AS release_index, problem.value AS issue +FROM published p, json_each(json_array( + CASE WHEN p.type NOT IN ('mod','module','theme','payment-gateway','server-manager','domain-registrar','translation') + THEN 'type: choose a supported manifest type (mod will become module)' END, + CASE WHEN p.manifest_license IS NULL OR length(trim(p.manifest_license)) NOT BETWEEN 1 AND 1000 OR p.manifest_license GLOB '*[^A-Za-z0-9.()+_: |-]*' + THEN 'license: supply a valid manifest license string or legacy spdx_id/name' END, + CASE WHEN length(trim(p.author_name)) = 0 OR length(p.author_name) > 200 + OR (p.author_url IS NOT NULL AND p.author_url != '' AND (length(p.author_url) > 2048 OR (p.author_url NOT LIKE 'https://%' AND p.author_url NOT LIKE 'http://%'))) + THEN 'author: correct the developer name/URL before copying package attribution' END, + CASE WHEN CASE WHEN NOT json_valid(p.releases) THEN 1 WHEN json_type(p.releases) != 'array' THEN 1 ELSE json_array_length(p.releases) NOT BETWEEN 1 AND 100 END + THEN 'releases: supply a non-empty JSON array with at most 100 releases' END, + CASE WHEN (SELECT COUNT(*) FROM release_rows r WHERE r.id=p.id AND json_extract(r.release,'$.tag') IN (p.version,'v'||p.version)) != 1 + THEN 'version: identify exactly one current release matching version or v+version' END +)) problem WHERE problem.value IS NOT NULL +UNION ALL +SELECT id, release_index, 'requires: set explicit FOSSBilling and PHP constraints for this release; missing values are not guessed' +FROM release_rows WHERE release_type != 'object' + OR COALESCE(json_type(release,'$.requires'),'') != 'object' + OR COALESCE(json_type(release,'$.requires.fossbilling'),'') != 'text' + OR COALESCE(json_type(release,'$.requires.php'),'') != 'text' + OR COALESCE(length(trim(json_extract(release,'$.requires.fossbilling'))),0) NOT BETWEEN 1 AND 1000 + OR COALESCE(length(trim(json_extract(release,'$.requires.php'))),0) NOT BETWEEN 1 AND 1000 + OR EXISTS (SELECT 1 FROM json_each(CASE WHEN json_type(release,'$.requires') = 'object' THEN json_extract(release,'$.requires') ELSE '{}' END) + WHERE key NOT IN ('fossbilling','php','modules')) + OR (json_type(release,'$.requires.modules') IS NOT NULL AND json_type(release,'$.requires.modules') != 'object') + OR EXISTS (SELECT 1 FROM json_each(CASE WHEN json_type(release,'$.requires.modules') = 'object' THEN json_extract(release,'$.requires.modules') ELSE '{}' END) + WHERE key = '' OR key GLOB '*[^a-z0-9_-]*' OR type != 'text' OR length(trim(value)) NOT BETWEEN 1 AND 1000) +ORDER BY id, release_index, issue; diff --git a/src/services/extensions/v2/db/migrations/0029_align_manifest.sql b/src/services/extensions/v2/db/migrations/0029_align_manifest.sql index 7e0a8cd6..24759807 100644 --- a/src/services/extensions/v2/db/migrations/0029_align_manifest.sql +++ b/src/services/extensions/v2/db/migrations/0029_align_manifest.sql @@ -1,5 +1,63 @@ -- Preserve physical website/license storage for v1, while v2 exposes manifest names. -- Reviewed revision bodies remain an audit log in their original format. +-- Block before changing data; run db/manifest-preflight.sql for the affected +-- extension IDs and release indexes, correct those rows, then retry. +CREATE VIEW _manifest_migration_issues AS +WITH published AS ( + SELECT e.*, d.name AS author_name, d.url AS author_url, + CASE WHEN json_valid(e.license) THEN CASE json_type(e.license) + WHEN 'text' THEN json_extract(e.license,'$') + WHEN 'object' THEN CASE + WHEN json_type(e.license,'$.spdx_id') = 'text' THEN json_extract(e.license,'$.spdx_id') + WHEN json_type(e.license,'$.spdx_id') IS NULL OR json_type(e.license,'$.spdx_id') = 'null' + THEN CASE WHEN json_type(e.license,'$.name') = 'text' THEN json_extract(e.license,'$.name') END + END END END AS manifest_license + FROM extensions e JOIN developers d ON d.id = e.developer_id + WHERE e.published_at IS NOT NULL +), release_rows AS ( + SELECT p.id, r.key AS release_index, r.type AS release_type, + CASE WHEN r.type = 'object' THEN r.value ELSE '{}' END AS release + FROM published p, json_each(CASE WHEN json_valid(p.releases) THEN p.releases ELSE '[]' END) r +) +SELECT p.id, NULL AS release_index, problem.value AS issue +FROM published p, json_each(json_array( + CASE WHEN p.type NOT IN ('mod','module','theme','payment-gateway','server-manager','domain-registrar','translation') + THEN 'type: choose a supported manifest type (mod will become module)' END, + CASE WHEN p.manifest_license IS NULL OR length(trim(p.manifest_license)) NOT BETWEEN 1 AND 1000 OR p.manifest_license GLOB '*[^A-Za-z0-9.()+_: |-]*' + THEN 'license: supply a valid manifest license string or legacy spdx_id/name' END, + CASE WHEN length(trim(p.author_name)) = 0 OR length(p.author_name) > 200 + OR (p.author_url IS NOT NULL AND p.author_url != '' AND (length(p.author_url) > 2048 OR (p.author_url NOT LIKE 'https://%' AND p.author_url NOT LIKE 'http://%'))) + THEN 'author: correct the developer name/URL before copying package attribution' END, + CASE WHEN CASE WHEN NOT json_valid(p.releases) THEN 1 WHEN json_type(p.releases) != 'array' THEN 1 ELSE json_array_length(p.releases) NOT BETWEEN 1 AND 100 END + THEN 'releases: supply a non-empty JSON array with at most 100 releases' END, + CASE WHEN (SELECT COUNT(*) FROM release_rows r WHERE r.id=p.id AND json_extract(r.release,'$.tag') IN (p.version,'v'||p.version)) != 1 + THEN 'version: identify exactly one current release matching version or v+version' END +)) problem WHERE problem.value IS NOT NULL +UNION ALL +SELECT id, release_index, 'requires: set explicit FOSSBilling and PHP constraints for this release; missing values are not guessed' +FROM release_rows WHERE release_type != 'object' + OR COALESCE(json_type(release,'$.requires'),'') != 'object' + OR COALESCE(json_type(release,'$.requires.fossbilling'),'') != 'text' + OR COALESCE(json_type(release,'$.requires.php'),'') != 'text' + OR COALESCE(length(trim(json_extract(release,'$.requires.fossbilling'))),0) NOT BETWEEN 1 AND 1000 + OR COALESCE(length(trim(json_extract(release,'$.requires.php'))),0) NOT BETWEEN 1 AND 1000 + OR EXISTS (SELECT 1 FROM json_each(CASE WHEN json_type(release,'$.requires') = 'object' THEN json_extract(release,'$.requires') ELSE '{}' END) + WHERE key NOT IN ('fossbilling','php','modules')) + OR (json_type(release,'$.requires.modules') IS NOT NULL AND json_type(release,'$.requires.modules') != 'object') + OR EXISTS (SELECT 1 FROM json_each(CASE WHEN json_type(release,'$.requires.modules') = 'object' THEN json_extract(release,'$.requires.modules') ELSE '{}' END) + WHERE key = '' OR key GLOB '*[^a-z0-9_-]*' OR type != 'text' OR length(trim(value)) NOT BETWEEN 1 AND 1000) +ORDER BY id, release_index, issue; +--> statement-breakpoint +CREATE TABLE _manifest_migration_check ( + invalid_count INTEGER CONSTRAINT manifest_data_requires_manual_correction CHECK (invalid_count = 0) +); +--> statement-breakpoint +INSERT INTO _manifest_migration_check SELECT COUNT(*) FROM _manifest_migration_issues; +--> statement-breakpoint +DROP TABLE _manifest_migration_check; +--> statement-breakpoint +DROP VIEW _manifest_migration_issues; +--> statement-breakpoint ALTER TABLE extensions ADD COLUMN manifest TEXT; --> statement-breakpoint DROP TRIGGER resource_extensions_insert; @@ -148,29 +206,23 @@ CREATE TRIGGER resource_extensions_delete AFTER DELETE ON extensions BEGIN END; --> statement-breakpoint UPDATE extensions SET - type = CASE WHEN type IN ('mod','hook') THEN 'module' ELSE type END, - license = CASE WHEN json_valid(license) THEN - CASE WHEN json_type(license) = 'object' THEN json_quote(COALESCE(json_extract(license,'$.spdx_id'),json_extract(license,'$.name'),'Unavailable')) ELSE license END - ELSE json_quote(COALESCE(license,'Unavailable')) END, + type = CASE WHEN type = 'mod' THEN 'module' ELSE type END, + license = json_quote(CASE WHEN json_type(license) = 'object' + THEN COALESCE(json_extract(license,'$.spdx_id'),json_extract(license,'$.name')) + ELSE json_extract(license,'$') END), manifest = json_object( 'manifest_version', 1, 'author', json((SELECT CASE WHEN url IS NULL OR url = '' THEN json_object('name',name) ELSE json_object('name',name,'url',url) END FROM developers WHERE developers.id=extensions.developer_id)), - 'requires', json_object('fossbilling', COALESCE( - (SELECT '>=' || json_extract(value,'$.min_fossbilling_version') FROM json_each(CASE WHEN json_valid(extensions.releases) THEN extensions.releases ELSE '[]' END) - WHERE json_extract(value,'$.min_fossbilling_version') IS NOT NULL - ORDER BY CASE WHEN json_extract(value,'$.tag') IN (extensions.version,'v'||extensions.version) THEN 0 ELSE 1 END, key LIMIT 1), '*'), 'php','*') + 'requires', json((SELECT json_extract(value,'$.requires') FROM json_each(extensions.releases) + WHERE json_extract(value,'$.tag') IN (extensions.version,'v'||extensions.version))) ) WHERE published_at IS NOT NULL; - --> statement-breakpoint --- Preserve every release entry while replacing the lossy minimum-version field. +-- Requirements have been explicitly reviewed. Remove only the obsolete v1 field. UPDATE extensions SET releases = ( - SELECT json_group_array(json(json_set(json_remove(value,'$.min_fossbilling_version'), '$.requires', - json(CASE WHEN json_type(value,'$.requires') = 'object' THEN json_extract(value,'$.requires') - ELSE json_object('fossbilling',CASE WHEN json_extract(value,'$.min_fossbilling_version') IS NOT NULL - THEN '>=' || json_extract(value,'$.min_fossbilling_version') ELSE '*' END,'php','*') END)))) + SELECT json_group_array(json(json_remove(value,'$.min_fossbilling_version'))) FROM json_each(extensions.releases) -) WHERE published_at IS NOT NULL AND json_valid(releases) AND json_type(releases)='array'; +) WHERE published_at IS NOT NULL; --> statement-breakpoint DROP INDEX idx_extensions_catalogue_order; --> statement-breakpoint diff --git a/test/services/extensions/v1/index.test.ts b/test/services/extensions/v1/index.test.ts index 22b0bacf..f1dbe939 100644 --- a/test/services/extensions/v1/index.test.ts +++ b/test/services/extensions/v1/index.test.ts @@ -16,7 +16,7 @@ import { applyTestMigrations } from "../../../utils/apply-migrations"; const testExtensionRows = [ { id: "Example", - type: "mod" as const, + type: "module" as const, developerId: "fossbilling", name: "Example Module", description: "An example module for developers.", @@ -28,7 +28,7 @@ const testExtensionRows = [ "https://github.com/FOSSBilling/example-module/releases/download/0.0.5/Example.zip", changelog_url: "https://github.com/FOSSBilling/example-module/releases/tag/0.0.5", - min_fossbilling_version: "0.6" + requires: { fossbilling: ">=0.6.0", php: ">=8.3" } }, { tag: "0.0.4", @@ -37,14 +37,11 @@ const testExtensionRows = [ "https://github.com/FOSSBilling/example-module/releases/download/0.0.4/Example.zip", changelog_url: "https://github.com/FOSSBilling/example-module/releases/tag/0.0.4", - min_fossbilling_version: "0.5" + requires: { fossbilling: ">=0.5.0", php: ">=8.2" } } ]), website: "https://fossbilling.org", - license: JSON.stringify({ - name: "Apache 2.0", - URL: "https://www.apache.org/licenses/LICENSE-2.0" - }), + license: JSON.stringify("Apache-2.0"), iconUrl: "https://raw.githubusercontent.com/FOSSBilling/example-module/main/src/icon.svg", readme: "# Example module\n\nThis is an example module.", @@ -67,11 +64,11 @@ const testExtensionRows = [ tag: "1.0.0", date: "2024-01-01T00:00:00Z", download_url: "https://example.com/TestTheme.zip", - min_fossbilling_version: "0.6" + requires: { fossbilling: ">=0.6.0", php: ">=8.3" } } ]), website: "https://fossbilling.org", - license: JSON.stringify({ name: "MIT" }), + license: JSON.stringify("MIT"), readme: "# Test Theme", source: JSON.stringify({ type: "github", repo: "FOSSBilling/test-theme" }), version: "1.0.0", @@ -111,6 +108,23 @@ describe("Extensions API v1", () => { ); }); + it("projects canonical licenses and release requirements into the v1 contract", async () => { + const ctx = createExecutionContext(); + const response = await app.request("/extensions/v1/Example", {}, env, ctx); + await waitOnExecutionContext(ctx); + const body = (await response.json()) as { + result: { + type: string; + license: { name: string }; + releases: Array>; + }; + }; + expect(body.result.type).toBe("mod"); + expect(body.result.license).toEqual({ name: "Apache-2.0" }); + expect(body.result.releases[0].min_fossbilling_version).toBe("0.6.0"); + expect(body.result.releases[0]).not.toHaveProperty("requires"); + }); + describe("GET /list", () => { it("should return all extensions", async () => { const ctx = createExecutionContext(); @@ -440,7 +454,7 @@ describe("Extensions API v1", () => { expect(res.status).toBe(200); const svg = await res.text(); - expect(svg).toContain("Apache 2.0"); + expect(svg).toContain("Apache-2.0"); }); it("should return red SVG for unknown badge type", async () => { @@ -486,3 +500,50 @@ describe("Extensions API v1", () => { }); }); }); + +describe("v1 projection failures", () => { + beforeAll(applyTestMigrations); + it.each(["not-a-range", ">=2.0.0 <1.0.0"])( + "flags a requirement with no v1 minimum: %s", + async (constraint) => { + const db = getExtensionsDb(env.DB_EXTENSIONS); + await db.delete(extensions); + await db.delete(developers); + await db.insert(developers).values({ + id: "projection-owner", + type: "organization", + name: "Owner" + }); + const original = JSON.parse(testExtensionRows[0].releases); + original[0].requires.fossbilling = constraint; + await db.insert(extensions).values({ + ...testExtensionRows[0], + developerId: "projection-owner", + releases: JSON.stringify(original), + publishedAt: "2026-01-01T00:00:00Z" + }); + const ctx = createExecutionContext(); + const response = await app.request( + "/extensions/v1/Example", + {}, + env, + ctx + ); + await waitOnExecutionContext(ctx); + expect(response.status).toBe(500); + await expect(response.json()).resolves.toMatchObject({ + error: { + message: expect.stringContaining("requires manual correction") + } + }); + const badge = await app.request( + "/extensions/v1/Example/badges/min_fossbilling_version", + {}, + env, + ctx + ); + expect(badge.status).toBe(500); + expect(await badge.text()).not.toContain("v0.0.0"); + } + ); +}); diff --git a/test/services/extensions/v2/account.test.ts b/test/services/extensions/v2/account.test.ts index e19a1eaf..8adb66d9 100644 --- a/test/services/extensions/v2/account.test.ts +++ b/test/services/extensions/v2/account.test.ts @@ -248,13 +248,13 @@ describe("Extensions API v2", () => { }); await insertExtension(db, { id: "account-extension", - type: "mod", + type: "module", developer_id: "account-developer", name: "Account Extension", description: "description", releases: "[]", website: "https://example.com", - license: '{"name":"MIT"}', + license: '"MIT"', icon_url: null, readme: "# Readme", source: '{"type":"github","repo":"example/account"}', diff --git a/test/services/extensions/v2/db-fixtures.ts b/test/services/extensions/v2/db-fixtures.ts index 8cc0db9d..9505f9b2 100644 --- a/test/services/extensions/v2/db-fixtures.ts +++ b/test/services/extensions/v2/db-fixtures.ts @@ -31,6 +31,7 @@ export interface ExtensionRow { developer_id: string; published_at: string | null; published_revision_id: string | null; + manifest: string | null; type: string | null; name: string | null; description: string | null; @@ -266,8 +267,8 @@ export async function insertExtension( (id, developer_id, published_at, published_revision_id, type, name, description, releases, website, license, icon_url, readme, source, version, download_url, created_at, updated_at, delisted_at, - delist_reason) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + delist_reason, manifest) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` ) .bind( row.id, @@ -279,7 +280,7 @@ export async function insertExtension( row.description ?? "d", row.releases ?? "[]", row.website ?? "https://e.com", - row.license ?? '{"name":"MIT"}', + row.license ?? '"MIT"', row.icon_url ?? null, row.readme ?? "r", row.source ?? '{"type":"github","repo":"example/ext"}', @@ -288,7 +289,13 @@ export async function insertExtension( row.created_at ?? now, row.updated_at ?? now, row.delisted_at ?? null, - row.delist_reason ?? null + row.delist_reason ?? null, + row.manifest ?? + JSON.stringify({ + manifest_version: 1, + author: { name: "Developer" }, + requires: { fossbilling: ">=0.6.0", php: ">=8.3" } + }) ) .run(); } diff --git a/test/services/extensions/v2/migrations.test.ts b/test/services/extensions/v2/migrations.test.ts index 38b309af..e7e1f1cb 100644 --- a/test/services/extensions/v2/migrations.test.ts +++ b/test/services/extensions/v2/migrations.test.ts @@ -1012,10 +1012,36 @@ it("migrates published manifest metadata without changing reviewed audit bodies" db.prepare( "INSERT INTO extension_revisions(id,extension_id,developer_id,submitted_by,status,content) VALUES('audit','legacy','dev','owner','approved',?)" ).run(audit); + const preflight = readFileSync( + join(migrationsDirectory, "../manifest-preflight.sql"), + "utf8" + ); + expect(db.prepare(preflight).all()).toEqual([ + { + id: "legacy", + release_index: 0, + issue: expect.stringContaining("set explicit FOSSBilling and PHP") + } + ]); + db.exec("BEGIN"); + expect(() => db.exec(migration("0029_align_manifest.sql"))).toThrow( + /manifest_data_requires_manual_correction/ + ); + db.exec("ROLLBACK"); + expect(columnNames(db, "extensions")).not.toContain("manifest"); + expect( + db.prepare("SELECT type FROM extensions WHERE id='legacy'").get()?.type + ).toBe("mod"); + // Operator supplies actual requirements, not guessed wildcard metadata. + db.prepare( + "UPDATE extensions SET releases=json_set(releases,'$[0].requires',json(?)) WHERE id='legacy'" + ).run(JSON.stringify({ fossbilling: ">=0.8.9", php: ">=8.3" })); + expect(db.prepare(preflight).all()).toEqual([]); // Existing over-quota collections must survive a structural backfill. db.exec( "UPDATE extension_resource_usage SET extensions=101 WHERE scope='developer'" ); + db.exec("BEGIN"); db.exec(migration("0029_align_manifest.sql")); db.exec("COMMIT"); @@ -1044,7 +1070,7 @@ it("migrates published manifest metadata without changing reviewed audit bodies" tag: "1.0.0", date: "2026-01-01", download_url: "https://example.com/a.zip", - requires: { fossbilling: ">=0.8.9", php: "*" } + requires: { fossbilling: ">=0.8.9", php: ">=8.3" } } ]); expect(row.type).toBe("module"); @@ -1052,7 +1078,7 @@ it("migrates published manifest metadata without changing reviewed audit bodies" expect(JSON.parse(row.manifest as string)).toEqual({ manifest_version: 1, author: { name: "Developer", url: "https://example.com" }, - requires: { fossbilling: ">=0.8.9", php: "*" } + requires: { fossbilling: ">=0.8.9", php: ">=8.3" } }); expect( db @@ -1095,3 +1121,96 @@ it("migrates published manifest metadata without changing reviewed audit bodies" db.close(); } }); + +it.each([ + ["license", "not-json", "license:", null], + ["license", '{"name":42}', "license:", null], + ["type", "hook", "type:", null], + ["releases", "not-json", "releases:", null], + [ + "releases", + JSON.stringify([ + { + tag: "1.0.0", + requires: { + fossbilling: ">=0.8.9", + php: ">=8.3", + modules: { branding: 42 } + } + } + ]), + "requires:", + 0 + ], + [ + "releases", + JSON.stringify([{ tag: "1.0.0", requires: { fossbilling: ">=0.8.9" } }]), + "requires:", + 0 + ], + [ + "releases", + JSON.stringify([ + { tag: "1.0.0", requires: { fossbilling: "*", php: "*" } }, + { tag: "v1.0.0", requires: { fossbilling: "*", php: "*" } } + ]), + "version:", + null + ] +] as const)( + "flags %s data for manual correction before migration", + (column, value, issue, releaseIndex) => { + const db = new DatabaseSync(":memory:"); + try { + db.exec("PRAGMA foreign_keys=ON"); + for (const name of migrationNames.filter((name) => name < "0029")) + db.exec(migration(name)); + db.exec( + "INSERT INTO developers(id,type,name) VALUES('dev','organization','Developer')" + ); + const releases = JSON.stringify([ + { + tag: "1.0.0", + date: "2026-01-01", + download_url: "https://example.com/release.zip", + requires: { fossbilling: ">=0.8.9", php: ">=8.3" } + } + ]); + db.prepare( + `INSERT INTO extensions(id,developer_id,type,name,description,releases,website,license,readme,source,version,download_url,published_at) + VALUES('needs-review','dev','mod','Example','Description',?,'https://example.com','{"name":"MIT"}','README','{}','1.0.0','https://example.com/release.zip','now')` + ).run(releases); + db.prepare( + `UPDATE extensions SET ${column}=? WHERE id='needs-review'` + ).run(value); + const preflight = readFileSync( + join(migrationsDirectory, "../manifest-preflight.sql"), + "utf8" + ); + expect(db.prepare(preflight).all()).toEqual( + expect.arrayContaining([ + { + id: "needs-review", + release_index: releaseIndex, + issue: expect.stringContaining(issue) + } + ]) + ); + db.exec("BEGIN"); + expect(() => db.exec(migration("0029_align_manifest.sql"))).toThrow( + /manifest_data_requires_manual_correction/ + ); + db.exec("ROLLBACK"); + expect(columnNames(db, "extensions")).not.toContain("manifest"); + expect( + db + .prepare( + `SELECT ${column} AS value FROM extensions WHERE id='needs-review'` + ) + .get()?.value + ).toBe(value); + } finally { + db.close(); + } + } +); diff --git a/test/services/extensions/v2/public-extensions.test.ts b/test/services/extensions/v2/public-extensions.test.ts index 449019eb..0c3e4302 100644 --- a/test/services/extensions/v2/public-extensions.test.ts +++ b/test/services/extensions/v2/public-extensions.test.ts @@ -29,7 +29,7 @@ describe("Extensions API v2", () => { description: `Description for ${id}`, releases: '[{"tag":"1.0.0"}]', website: "https://example.com", - license: '{"name":"MIT"}', + license: '"MIT"', icon_url: null, readme: `README for ${id}`, source: '{"type":"github","repo":"example/extensions"}', diff --git a/test/services/extensions/v2/resource-bounds.test.ts b/test/services/extensions/v2/resource-bounds.test.ts index 8f37b28e..e189a9e5 100644 --- a/test/services/extensions/v2/resource-bounds.test.ts +++ b/test/services/extensions/v2/resource-bounds.test.ts @@ -241,7 +241,7 @@ describe("Extension resource admission", () => { "Résumé 🧪", JSON.stringify(content.releases), "https://example.test/é", - JSON.stringify({ name: "Lïcence" }), + JSON.stringify("Lïcence"), "https://example.test/😀.png", "読んでください", JSON.stringify({ type: "custom", repo: "café/😀" }), @@ -1111,7 +1111,7 @@ describe("Bounded revision reads and maintenance", () => { it("preserves under-limit legacy fields in public and owner details", async () => { await owned(); const website = "https://example.test/" + "x".repeat(3000); - const license = { name: "L".repeat(5000) }; + const license = "L".repeat(5000); const source = { type: "custom", repo: "R".repeat(5000) }; await db .prepare( @@ -1133,14 +1133,14 @@ describe("Bounded revision reads and maintenance", () => { expect(publicDetail.error).toBeNull(); expect(publicDetail.data).toMatchObject({ homepage_url: website, - license: license.name, + license, source }); const ownerDetail = await extensions.getOwned("live"); expect(ownerDetail.error).toBeNull(); expect(ownerDetail.data?.extension.published).toMatchObject({ homepage_url: website, - license: license.name, + license, source }); }); From 2e8046d9935295215806ecf40634c32b9f4b912f Mon Sep 17 00:00:00 2001 From: Adam Daley Date: Wed, 4 Nov 2026 03:19:42 +0000 Subject: [PATCH 3/5] Address manifest contract review findings --- src/services/extensions/v1/list-cache.ts | 2 +- .../extensions/v2/db/manifest-preflight.sql | 5 +- .../v2/db/migrations/0029_align_manifest.sql | 16 ++++- src/services/extensions/v2/db/schema.ts | 2 + .../extensions/v2/routes/public-extensions.ts | 2 +- .../extensions/v2/schemas/extensions.ts | 4 +- test/services/extensions/v1/index.test.ts | 10 +++ test/services/extensions/v2/manifest.test.ts | 64 +++++++++++++++++- .../services/extensions/v2/migrations.test.ts | 66 +++++++++++++++---- .../v2/moderation-revalidate.test.ts | 17 +++++ .../extensions/v2/resource-bounds.test.ts | 23 +++++++ 11 files changed, 191 insertions(+), 20 deletions(-) diff --git a/src/services/extensions/v1/list-cache.ts b/src/services/extensions/v1/list-cache.ts index c3111824..6e276f8b 100644 --- a/src/services/extensions/v1/list-cache.ts +++ b/src/services/extensions/v1/list-cache.ts @@ -33,7 +33,7 @@ export function invalidateListCache( // Mirrors listCacheKey's encoding: the "all" sentinel for the unfiltered // list, "%"-prefixed encoded keys per type. const keys = [listCacheKey(undefined, undefined)].concat( - EXTENSION_TYPES.map((type) => listCacheKey(type, undefined)) + [...EXTENSION_TYPES, "mod"].map((type) => listCacheKey(type, undefined)) ); for (const key of keys) { waitUntil(kv.delete(key).catch(() => {})); diff --git a/src/services/extensions/v2/db/manifest-preflight.sql b/src/services/extensions/v2/db/manifest-preflight.sql index 815e8df8..b4811e20 100644 --- a/src/services/extensions/v2/db/manifest-preflight.sql +++ b/src/services/extensions/v2/db/manifest-preflight.sql @@ -23,7 +23,10 @@ FROM published p, json_each(json_array( CASE WHEN p.manifest_license IS NULL OR length(trim(p.manifest_license)) NOT BETWEEN 1 AND 1000 OR p.manifest_license GLOB '*[^A-Za-z0-9.()+_: |-]*' THEN 'license: supply a valid manifest license string or legacy spdx_id/name' END, CASE WHEN length(trim(p.author_name)) = 0 OR length(p.author_name) > 200 - OR (p.author_url IS NOT NULL AND p.author_url != '' AND (length(p.author_url) > 2048 OR (p.author_url NOT LIKE 'https://%' AND p.author_url NOT LIKE 'http://%'))) + OR (p.author_url IS NOT NULL AND p.author_url != '' AND (length(p.author_url) > 2048 + OR (p.author_url NOT LIKE 'https://%' AND p.author_url NOT LIKE 'http://%') + OR substr(p.author_url,instr(p.author_url,'://')+3,1) IN ('','/','?','#') + OR p.author_url GLOB ('*[' || char(9) || char(10) || char(13) || ' ]*'))) THEN 'author: correct the developer name/URL before copying package attribution' END, CASE WHEN CASE WHEN NOT json_valid(p.releases) THEN 1 WHEN json_type(p.releases) != 'array' THEN 1 ELSE json_array_length(p.releases) NOT BETWEEN 1 AND 100 END THEN 'releases: supply a non-empty JSON array with at most 100 releases' END, diff --git a/src/services/extensions/v2/db/migrations/0029_align_manifest.sql b/src/services/extensions/v2/db/migrations/0029_align_manifest.sql index 24759807..4f3ed737 100644 --- a/src/services/extensions/v2/db/migrations/0029_align_manifest.sql +++ b/src/services/extensions/v2/db/migrations/0029_align_manifest.sql @@ -26,7 +26,10 @@ FROM published p, json_each(json_array( CASE WHEN p.manifest_license IS NULL OR length(trim(p.manifest_license)) NOT BETWEEN 1 AND 1000 OR p.manifest_license GLOB '*[^A-Za-z0-9.()+_: |-]*' THEN 'license: supply a valid manifest license string or legacy spdx_id/name' END, CASE WHEN length(trim(p.author_name)) = 0 OR length(p.author_name) > 200 - OR (p.author_url IS NOT NULL AND p.author_url != '' AND (length(p.author_url) > 2048 OR (p.author_url NOT LIKE 'https://%' AND p.author_url NOT LIKE 'http://%'))) + OR (p.author_url IS NOT NULL AND p.author_url != '' AND (length(p.author_url) > 2048 + OR (p.author_url NOT LIKE 'https://%' AND p.author_url NOT LIKE 'http://%') + OR substr(p.author_url,instr(p.author_url,'://')+3,1) IN ('','/','?','#') + OR p.author_url GLOB ('*[' || char(9) || char(10) || char(13) || ' ]*'))) THEN 'author: correct the developer name/URL before copying package attribution' END, CASE WHEN CASE WHEN NOT json_valid(p.releases) THEN 1 WHEN json_type(p.releases) != 'array' THEN 1 ELSE json_array_length(p.releases) NOT BETWEEN 1 AND 100 END THEN 'releases: supply a non-empty JSON array with at most 100 releases' END, @@ -295,3 +298,14 @@ CREATE TRIGGER resource_extensions_update AFTER UPDATE OF type,name,description, ) THEN RAISE(ABORT,'extension_resource_quota') END; END; + +--> statement-breakpoint +-- The existing published-content CHECK predates this column. Enforce its +-- presence without rebuilding the parent of extension_revisions. +CREATE TRIGGER extensions_published_manifest_insert BEFORE INSERT ON extensions +WHEN NEW.published_at IS NOT NULL AND NEW.manifest IS NULL +BEGIN SELECT RAISE(ABORT,'extensions_published_manifest_required'); END; +--> statement-breakpoint +CREATE TRIGGER extensions_published_manifest_update BEFORE UPDATE OF published_at,manifest ON extensions +WHEN NEW.published_at IS NOT NULL AND NEW.manifest IS NULL +BEGIN SELECT RAISE(ABORT,'extensions_published_manifest_required'); END; diff --git a/src/services/extensions/v2/db/schema.ts b/src/services/extensions/v2/db/schema.ts index 58319588..8a1464e9 100644 --- a/src/services/extensions/v2/db/schema.ts +++ b/src/services/extensions/v2/db/schema.ts @@ -55,6 +55,8 @@ export const extensions = sqliteTable( // need. NULL for rows adopted from the pre-v2 directory, which were never // published through a revision. publishedRevisionId: text("published_revision_id"), + // 0029 enforces presence on published rows with insert/update triggers; + // extending the original CHECK would require rebuilding this FK parent. manifest: text("manifest"), type: text("type"), name: text("name"), diff --git a/src/services/extensions/v2/routes/public-extensions.ts b/src/services/extensions/v2/routes/public-extensions.ts index 64b22712..39796fdb 100644 --- a/src/services/extensions/v2/routes/public-extensions.ts +++ b/src/services/extensions/v2/routes/public-extensions.ts @@ -36,7 +36,7 @@ export function registerPublicExtensionsRoutes(app: ExtensionsV2App): void { } }, description: - "Extensions matching the scope: extension cards for scope=public, owned rows for scope=mine/all" + "Extensions matching the scope: extension cards for scope=public, owned rows for scope=mine, moderation rows for scope=all" }, 401: errorResponse("Invalid bearer token"), 403: errorResponse( diff --git a/src/services/extensions/v2/schemas/extensions.ts b/src/services/extensions/v2/schemas/extensions.ts index 4c09c342..f40db85b 100644 --- a/src/services/extensions/v2/schemas/extensions.ts +++ b/src/services/extensions/v2/schemas/extensions.ts @@ -77,7 +77,7 @@ export type ExtensionContent = z.infer; // reads, with a 200-code-unit ceiling so supplementary characters still fit. const StoredReleaseSchema = ReleaseSchema.extend({ requires: ManifestRequirementsSchema.optional(), - min_fossbilling_version: z.string().optional(), + min_fossbilling_version: z.string().min(1).max(100).optional(), tag: z .string() .min(1) @@ -145,6 +145,8 @@ const PublishedContentSchema = ExtensionContentSchema.extend({ description: z.string().min(1).max(4000), version: z.string().min(1).max(100), license: z.string().min(1), + homepage_url: z.string().min(1).optional(), + download_url: z.string().min(1), icon_url: z.string().optional() }); diff --git a/test/services/extensions/v1/index.test.ts b/test/services/extensions/v1/index.test.ts index f1dbe939..2af7102f 100644 --- a/test/services/extensions/v1/index.test.ts +++ b/test/services/extensions/v1/index.test.ts @@ -16,6 +16,11 @@ import { applyTestMigrations } from "../../../utils/apply-migrations"; const testExtensionRows = [ { id: "Example", + manifest: JSON.stringify({ + manifest_version: 1, + author: { name: "FOSSBilling" }, + requires: { fossbilling: ">=0.6.0", php: ">=8.3" } + }), type: "module" as const, developerId: "fossbilling", name: "Example Module", @@ -55,6 +60,11 @@ const testExtensionRows = [ }, { id: "TestTheme", + manifest: JSON.stringify({ + manifest_version: 1, + author: { name: "FOSSBilling" }, + requires: { fossbilling: ">=0.6.0", php: ">=8.3" } + }), type: "theme" as const, developerId: "fossbilling", name: "Test Theme", diff --git a/test/services/extensions/v2/manifest.test.ts b/test/services/extensions/v2/manifest.test.ts index 00d82dc6..a310b855 100644 --- a/test/services/extensions/v2/manifest.test.ts +++ b/test/services/extensions/v2/manifest.test.ts @@ -10,7 +10,8 @@ import { sampleContent, db } from "./harness"; -import { insertUser } from "./db-fixtures"; +import { StoredExtensionContentSchema } from "../../../../src/services/extensions/v2/schemas/extensions"; +import { insertExtension, insertUser } from "./db-fixtures"; vi.mock("@octokit/request", async () => (await import("../../../mocks/octokit")).octokitRequestMock() @@ -104,6 +105,66 @@ describe("manifest v1 publishing", () => { }); }); + it("keeps the historical minimum-version field bounded in revision schemas", () => { + const content = (minimum: string) => ({ + releases: [ + { ...sampleContent().releases[0], min_fossbilling_version: minimum } + ] + }); + expect( + StoredExtensionContentSchema.safeParse(content("x".repeat(100))).success + ).toBe(true); + const oversized = StoredExtensionContentSchema.safeParse( + content("x".repeat(101)) + ); + expect(oversized.success).toBe(false); + if (!oversized.success) + expect(oversized.error.issues).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + code: "too_big", + path: ["releases", 0, "min_fossbilling_version"] + }) + ]) + ); + }); + + it("rejects the retired website key specifically as an unknown root field", async () => { + await seedDeveloper("owner", "user-1"); + const response = await post( + "/extensions/v2/extensions", + await authHeaders("user-1"), + { ...sampleCreate(), website: "https://example.com" } + ); + expect(response.status).toBe(422); + await expect(response.json()).resolves.toMatchObject({ + error: { + details: expect.arrayContaining([ + expect.objectContaining({ + code: "unrecognized_keys", + keys: ["website"], + path: [] + }) + ]) + } + }); + }); + + it("allows existing long identities to be edited without relaxing new manifest IDs", async () => { + await seedDeveloper("owner", "user-1"); + const id = "e".repeat(120); + await insertExtension(db, { id, developer_id: "owner" }); + expect( + ( + await put( + `/extensions/v2/extensions/${id}`, + await authHeaders("user-1"), + sampleContent() + ) + ).status + ).toBe(202); + }); + it("accepts an omitted homepage and rejects old fields and unsafe package paths", async () => { await seedDeveloper("owner", "user-1"); const headers = await authHeaders("user-1"); @@ -112,7 +173,6 @@ describe("manifest v1 publishing", () => { { type: "mod" }, { manifest_version: 2 }, { license: { name: "MIT" } }, - { website: "https://example.com" }, { version: "v1.0.0" }, { version: "01.0.0" }, { version: "1.0.0+" + "x".repeat(100) }, diff --git a/test/services/extensions/v2/migrations.test.ts b/test/services/extensions/v2/migrations.test.ts index e7e1f1cb..7cc35eb7 100644 --- a/test/services/extensions/v2/migrations.test.ts +++ b/test/services/extensions/v2/migrations.test.ts @@ -985,13 +985,29 @@ it("migrates published manifest metadata without changing reviewed audit bodies" db.exec( "INSERT INTO developers(id,type,name,url,owner_user_id) VALUES('dev','organization','Developer','https://example.com','owner')" ); + const olderRelease = { + tag: "0.9.0", + date: "2025-12-01", + download_url: "https://example.com/older.zip", + requires: { + fossbilling: "^0.7", + php: "^8.2", + modules: { "old-module": "~1.0" } + } + }; + const currentRequirements = { + fossbilling: ">=0.8.9", + php: ">=8.3", + modules: { branding: "^2" } + }; const releases = JSON.stringify([ { tag: "1.0.0", date: "2026-01-01", download_url: "https://example.com/a.zip", min_fossbilling_version: "0.8.9" - } + }, + { ...olderRelease, min_fossbilling_version: "0.7.0" } ]); db.prepare( `INSERT INTO extensions(id,developer_id,type,name,description,releases,website,license,readme,source,version,download_url,published_at) @@ -1035,7 +1051,7 @@ it("migrates published manifest metadata without changing reviewed audit bodies" // Operator supplies actual requirements, not guessed wildcard metadata. db.prepare( "UPDATE extensions SET releases=json_set(releases,'$[0].requires',json(?)) WHERE id='legacy'" - ).run(JSON.stringify({ fossbilling: ">=0.8.9", php: ">=8.3" })); + ).run(JSON.stringify(currentRequirements)); expect(db.prepare(preflight).all()).toEqual([]); // Existing over-quota collections must survive a structural backfill. db.exec( @@ -1070,15 +1086,34 @@ it("migrates published manifest metadata without changing reviewed audit bodies" tag: "1.0.0", date: "2026-01-01", download_url: "https://example.com/a.zip", - requires: { fossbilling: ">=0.8.9", php: ">=8.3" } - } + requires: currentRequirements + }, + olderRelease ]); + expect(() => + db.exec("UPDATE extensions SET manifest=NULL WHERE id='legacy'") + ).toThrow(/extensions_published_manifest_required/); + db.exec("INSERT INTO extensions(id,developer_id) VALUES('draft','dev')"); + expect( + db.prepare("SELECT manifest FROM extensions WHERE id='draft'").get() + ?.manifest + ).toBeNull(); + expect(() => + db.exec( + "UPDATE extensions SET published_at='now',type='module',name='Draft',description='d',releases='[]',website='',license='\"MIT\"',readme='r',source='{}',version='1.0.0',download_url='https://example.com' WHERE id='draft'" + ) + ).toThrow(/extensions_published_manifest_required/); + expect(() => + db.exec( + "INSERT INTO extensions(id,developer_id,published_at,type,name,description,releases,website,license,readme,source,version,download_url) VALUES('missing','dev','now','module','Missing','d','[]','','\"MIT\"','r','{}','1.0.0','https://example.com')" + ) + ).toThrow(/extensions_published_manifest_required/); expect(row.type).toBe("module"); expect(JSON.parse(row.license as string)).toBe("Apache-2.0"); expect(JSON.parse(row.manifest as string)).toEqual({ manifest_version: 1, author: { name: "Developer", url: "https://example.com" }, - requires: { fossbilling: ">=0.8.9", php: ">=8.3" } + requires: currentRequirements }); expect( db @@ -1125,7 +1160,14 @@ it("migrates published manifest metadata without changing reviewed audit bodies" it.each([ ["license", "not-json", "license:", null], ["license", '{"name":42}', "license:", null], + ["license", "null", "license:", null], + ["license", "42", "license:", null], + ["license", "[]", "license:", null], ["type", "hook", "type:", null], + ["developer_url", "javascript:alert(1)", "author:", null], + ["developer_url", "https://", "author:", null], + ["developer_url", "https://bad host.example", "author:", null], + ["developer_url", "https://example.com/" + "x".repeat(2048), "author:", null], ["releases", "not-json", "releases:", null], [ "releases", @@ -1180,9 +1222,10 @@ it.each([ `INSERT INTO extensions(id,developer_id,type,name,description,releases,website,license,readme,source,version,download_url,published_at) VALUES('needs-review','dev','mod','Example','Description',?,'https://example.com','{"name":"MIT"}','README','{}','1.0.0','https://example.com/release.zip','now')` ).run(releases); - db.prepare( - `UPDATE extensions SET ${column}=? WHERE id='needs-review'` - ).run(value); + const table = column === "developer_url" ? "developers" : "extensions"; + const field = column === "developer_url" ? "url" : column; + const id = column === "developer_url" ? "dev" : "needs-review"; + db.prepare(`UPDATE ${table} SET ${field}=? WHERE id=?`).run(value, id); const preflight = readFileSync( join(migrationsDirectory, "../manifest-preflight.sql"), "utf8" @@ -1203,11 +1246,8 @@ it.each([ db.exec("ROLLBACK"); expect(columnNames(db, "extensions")).not.toContain("manifest"); expect( - db - .prepare( - `SELECT ${column} AS value FROM extensions WHERE id='needs-review'` - ) - .get()?.value + db.prepare(`SELECT ${field} AS value FROM ${table} WHERE id=?`).get(id) + ?.value ).toBe(value); } finally { db.close(); diff --git a/test/services/extensions/v2/moderation-revalidate.test.ts b/test/services/extensions/v2/moderation-revalidate.test.ts index cef63803..def2c6b6 100644 --- a/test/services/extensions/v2/moderation-revalidate.test.ts +++ b/test/services/extensions/v2/moderation-revalidate.test.ts @@ -4,6 +4,7 @@ import { setupExtensionsV2Tests, db, authHeaders, + get, post, put, del, @@ -82,6 +83,22 @@ async function delist(as: string): Promise { } describe("CDN cache revalidation on extension mutations", () => { + it("invalidates both v1 module cache aliases after a successful mutation", async () => { + await seedModAndExtension(); + for (const type of ["mod", "module"]) { + const response = await get(`/extensions/v1/list?type=${type}`, {}); + expect(response.status).toBe(200); + await expect(response.json()).resolves.toMatchObject({ + result: [{ id: "live-ext" }] + }); + } + expect((await delist("mod-1")).status).toBe(200); + for (const type of ["mod", "module"]) { + const response = await get(`/extensions/v1/list?type=${type}`, {}); + await expect(response.json()).resolves.toMatchObject({ result: [] }); + } + }); + it("does not revalidate unchanged profile replays", async () => { const fetcher = stubFrontend(); const headers = await authHeaders("noop-revalidate-owner"); diff --git a/test/services/extensions/v2/resource-bounds.test.ts b/test/services/extensions/v2/resource-bounds.test.ts index e189a9e5..6d031ced 100644 --- a/test/services/extensions/v2/resource-bounds.test.ts +++ b/test/services/extensions/v2/resource-bounds.test.ts @@ -12,6 +12,7 @@ import { inventoryExtensionRetention } from "../../../../src/services/extensions/v2/db/resource-maintenance"; import { + ExtensionSchema, ExtensionListItemSchema, ExtensionUpdateSchema, OwnedExtensionListItemSchema @@ -1144,6 +1145,28 @@ describe("Bounded revision reads and maintenance", () => { source }); }); + it("describes historical detail URLs without applying the card URL bound", async () => { + await owned(); + const homepage = "https://example.test/" + "x".repeat(3000); + const download = "https://example.test/" + "y".repeat(3000); + await db + .prepare( + "UPDATE extensions SET website=?,download_url=?,releases=? WHERE id='live'" + ) + .bind(homepage, download, JSON.stringify(sampleContent().releases)) + .run(); + const extensions = new ExtensionsDatabase(getExtensionsDb(db)); + const detail = await extensions.getById("live"); + expect(detail.error).toBeNull(); + expect(ExtensionSchema.safeParse(detail.data).success).toBe(true); + expect(detail.data).toMatchObject({ + homepage_url: homepage, + download_url: download + }); + const cards = await extensions.list({}); + expect(cards.data?.items[0].download_url).toBeNull(); + expect(cards.data?.items[0].homepage_url).toBeUndefined(); + }); it("limits maintenance work per invocation", async () => { await owned(); for (let i = 0; i <= MAINTENANCE_BATCH_SIZE; i++) From 13787c366b84a8a4fe377c742170a49d4ecc3333 Mon Sep 17 00:00:00 2001 From: Adam Daley Date: Wed, 4 Nov 2026 03:24:04 +0000 Subject: [PATCH 4/5] Support reviewed corrections during manifest migration --- src/services/extensions/v1/database.ts | 4 +- src/services/extensions/v2/README.md | 8 +- .../v2/db/migrations/0029_align_manifest.sql | 126 +++++++++--------- .../extensions/v2/schemas/manifest.ts | 12 +- test/services/extensions/v1/index.test.ts | 2 +- test/services/extensions/v2/manifest.test.ts | 9 ++ .../services/extensions/v2/migrations.test.ts | 39 ++++-- 7 files changed, 122 insertions(+), 78 deletions(-) diff --git a/src/services/extensions/v1/database.ts b/src/services/extensions/v1/database.ts index f127b745..681ac2b8 100644 --- a/src/services/extensions/v1/database.ts +++ b/src/services/extensions/v1/database.ts @@ -254,7 +254,9 @@ function toV1License(stored: string): { name: string } { function toV1Releases(stored: string): Release[] { return (JSON.parse(stored) as ManifestRelease[]).map( ({ requires, ...release }) => { - const range = validRange(requires.fossbilling); + const range = requires.fossbilling.trim() + ? validRange(requires.fossbilling) + : null; const minimum = range === null ? null : minVersion(range); if (!minimum) { throw new Error( diff --git a/src/services/extensions/v2/README.md b/src/services/extensions/v2/README.md index ffdedb98..73acdab2 100644 --- a/src/services/extensions/v2/README.md +++ b/src/services/extensions/v2/README.md @@ -15,7 +15,7 @@ Endpoints are not listed here. The service publishes its own contract: ## Manifest contract -New v2 writes use the [FOSSBilling manifest v1 fields](https://github.com/FOSSBilling/modules/blob/main/manifest.schema.json): `manifest_version: 1`, `type: "module"` (replacing `mod`; former `hook` entries become modules), `author: {name, url?}`, a string `license`, optional `homepage_url` (replacing `website`), and `requires: {fossbilling, php, modules?}`. IDs allow lowercase letters, digits, underscores, and hyphens, up to 64 characters. Names are limited to 100 characters, descriptions to 500, and versions use SemVer without a `v` prefix. +New v2 writes use the [FOSSBilling manifest v1 fields](https://github.com/FOSSBilling/modules/blob/main/manifest.schema.json): `manifest_version: 1`, `type: "module"` (replacing `mod`; standalone `hook` entries require manual reclassification), `author: {name, url?}`, a string `license`, optional `homepage_url` (replacing `website`), and `requires: {fossbilling, php, modules?}`. IDs allow lowercase letters, digits, underscores, and hyphens, up to 64 characters. Names are limited to 100 characters, descriptions to 500, and versions use SemVer without a `v` prefix. `icon_url` is an optional path relative to the package root, as in `icon.svg`; it cannot be an absolute URL, absolute path, or contain `..` or backslashes. Consumers resolve package assets against their trusted distribution source. Historical icons may still be absolute URLs. @@ -25,7 +25,7 @@ API publishing fields `readme`, `source`, `download_url`, and `releases` remain Run the read-only `db/manifest-preflight.sql` query before deployment. It reports extension IDs and zero-based release indexes needing manual correction. Every published release must have an explicitly reviewed `requires` object, including its PHP constraint; migration does not invent `*`, an author, or a license placeholder. Use the package manifest/release metadata to fill missing requirements. A deliberate `*` is allowed only when it accurately represents the reviewed requirement. -Migration `0029_align_manifest.sql` stops with `manifest_data_requires_manual_correction` before changing library data if the preflight reports issues. Correct those rows and retry. Once prepared, it converts `mod` to `module`, converts legacy license objects to strings (using `spdx_id`, otherwise `name`), copies the current release's exact requirements and the developer's attribution, and removes obsolete release minimum fields. Physical `website` storage remains for v1 compatibility. Reviewed and pending revision bodies retain their original audit content; old pending revisions must be resubmitted using the new contract before approval. Published reads expect canonical storage; historical audit reads remain separate. +Migration `0029_align_manifest.sql` stops with `manifest_data_requires_manual_correction` if the preflight reports issues, rolling back the entire migration. Correct those rows and retry. Once prepared, it converts `mod` to `module`, converts legacy license objects to strings (using `spdx_id`, otherwise `name`), copies the current release's exact requirements and the developer's attribution, and removes obsolete release minimum fields. Physical `website` storage remains for v1 compatibility. Reviewed and pending revision bodies retain their original audit content; old pending revisions must be resubmitted using the new contract before approval. Published reads expect canonical storage; historical audit reads remain separate. For example, after checking an affected package's actual requirements, an operator can add `requires` to one release without rewriting its remaining fields: @@ -38,6 +38,10 @@ WHERE id = 'affected-extension'; Use the ID, index, and verified constraints from the preflight investigation; the values above are examples. Re-run the query until it returns no rows. +For imported collections already over a retained-resource quota, the direct update above will be rejected because it grows stored content. Add the reviewed, ID-scoped correction statements at `-- MANUAL_MANIFEST_CORRECTIONS` in migration `0029` instead. This section runs inside D1's atomic migration transaction after the growth gate is temporarily relaxed, while byte-accounting triggers stay active. Preflight then validates the corrected library before backfill; an error rolls back every correction and schema change. The normal growth gate is restored before commit. With this path, the external preflight report will still list the uncorrected rows until execution; the migration reruns preflight after applying the reviewed corrections. Do not lower usage counters or leave production triggers disabled to force an update through. + +After applying the prepared migration, run the read-only `db/resource-inventory.sql` reconciliation query. It must return no discrepancies; investigate any returned subjects before reopening writes. The migration regression test covers correction of an over-quota collection, rollback when issues remain, restored quota enforcement, and reconciled byte totals. + Apply the migration and API deployment in a controlled cutover: the old Worker expects object licenses, while the new Worker requires the added metadata column. This changes the v2 request/response contract: regenerate the Extensions site's client and update CI publishers together with the API rollout. The modules publisher must stop translating `module` to `mod` and send the manifest fields directly. Package-relative icons use v1's empty-string fallback. The v1 adapter continues to return `mod`, `website`, object licenses, and minimum release versions; constraints are converted to their earliest matching SemVer version. A malformed, unsatisfiable, or Composer-only constraint with no representable v1 minimum produces an explicit error identifying the release for manual correction; it is never replaced with `0.0.0`. diff --git a/src/services/extensions/v2/db/migrations/0029_align_manifest.sql b/src/services/extensions/v2/db/migrations/0029_align_manifest.sql index 4f3ed737..f61266d5 100644 --- a/src/services/extensions/v2/db/migrations/0029_align_manifest.sql +++ b/src/services/extensions/v2/db/migrations/0029_align_manifest.sql @@ -1,66 +1,5 @@ -- Preserve physical website/license storage for v1, while v2 exposes manifest names. -- Reviewed revision bodies remain an audit log in their original format. --- Block before changing data; run db/manifest-preflight.sql for the affected --- extension IDs and release indexes, correct those rows, then retry. -CREATE VIEW _manifest_migration_issues AS -WITH published AS ( - SELECT e.*, d.name AS author_name, d.url AS author_url, - CASE WHEN json_valid(e.license) THEN CASE json_type(e.license) - WHEN 'text' THEN json_extract(e.license,'$') - WHEN 'object' THEN CASE - WHEN json_type(e.license,'$.spdx_id') = 'text' THEN json_extract(e.license,'$.spdx_id') - WHEN json_type(e.license,'$.spdx_id') IS NULL OR json_type(e.license,'$.spdx_id') = 'null' - THEN CASE WHEN json_type(e.license,'$.name') = 'text' THEN json_extract(e.license,'$.name') END - END END END AS manifest_license - FROM extensions e JOIN developers d ON d.id = e.developer_id - WHERE e.published_at IS NOT NULL -), release_rows AS ( - SELECT p.id, r.key AS release_index, r.type AS release_type, - CASE WHEN r.type = 'object' THEN r.value ELSE '{}' END AS release - FROM published p, json_each(CASE WHEN json_valid(p.releases) THEN p.releases ELSE '[]' END) r -) -SELECT p.id, NULL AS release_index, problem.value AS issue -FROM published p, json_each(json_array( - CASE WHEN p.type NOT IN ('mod','module','theme','payment-gateway','server-manager','domain-registrar','translation') - THEN 'type: choose a supported manifest type (mod will become module)' END, - CASE WHEN p.manifest_license IS NULL OR length(trim(p.manifest_license)) NOT BETWEEN 1 AND 1000 OR p.manifest_license GLOB '*[^A-Za-z0-9.()+_: |-]*' - THEN 'license: supply a valid manifest license string or legacy spdx_id/name' END, - CASE WHEN length(trim(p.author_name)) = 0 OR length(p.author_name) > 200 - OR (p.author_url IS NOT NULL AND p.author_url != '' AND (length(p.author_url) > 2048 - OR (p.author_url NOT LIKE 'https://%' AND p.author_url NOT LIKE 'http://%') - OR substr(p.author_url,instr(p.author_url,'://')+3,1) IN ('','/','?','#') - OR p.author_url GLOB ('*[' || char(9) || char(10) || char(13) || ' ]*'))) - THEN 'author: correct the developer name/URL before copying package attribution' END, - CASE WHEN CASE WHEN NOT json_valid(p.releases) THEN 1 WHEN json_type(p.releases) != 'array' THEN 1 ELSE json_array_length(p.releases) NOT BETWEEN 1 AND 100 END - THEN 'releases: supply a non-empty JSON array with at most 100 releases' END, - CASE WHEN (SELECT COUNT(*) FROM release_rows r WHERE r.id=p.id AND json_extract(r.release,'$.tag') IN (p.version,'v'||p.version)) != 1 - THEN 'version: identify exactly one current release matching version or v+version' END -)) problem WHERE problem.value IS NOT NULL -UNION ALL -SELECT id, release_index, 'requires: set explicit FOSSBilling and PHP constraints for this release; missing values are not guessed' -FROM release_rows WHERE release_type != 'object' - OR COALESCE(json_type(release,'$.requires'),'') != 'object' - OR COALESCE(json_type(release,'$.requires.fossbilling'),'') != 'text' - OR COALESCE(json_type(release,'$.requires.php'),'') != 'text' - OR COALESCE(length(trim(json_extract(release,'$.requires.fossbilling'))),0) NOT BETWEEN 1 AND 1000 - OR COALESCE(length(trim(json_extract(release,'$.requires.php'))),0) NOT BETWEEN 1 AND 1000 - OR EXISTS (SELECT 1 FROM json_each(CASE WHEN json_type(release,'$.requires') = 'object' THEN json_extract(release,'$.requires') ELSE '{}' END) - WHERE key NOT IN ('fossbilling','php','modules')) - OR (json_type(release,'$.requires.modules') IS NOT NULL AND json_type(release,'$.requires.modules') != 'object') - OR EXISTS (SELECT 1 FROM json_each(CASE WHEN json_type(release,'$.requires.modules') = 'object' THEN json_extract(release,'$.requires.modules') ELSE '{}' END) - WHERE key = '' OR key GLOB '*[^a-z0-9_-]*' OR type != 'text' OR length(trim(value)) NOT BETWEEN 1 AND 1000) -ORDER BY id, release_index, issue; ---> statement-breakpoint -CREATE TABLE _manifest_migration_check ( - invalid_count INTEGER CONSTRAINT manifest_data_requires_manual_correction CHECK (invalid_count = 0) -); ---> statement-breakpoint -INSERT INTO _manifest_migration_check SELECT COUNT(*) FROM _manifest_migration_issues; ---> statement-breakpoint -DROP TABLE _manifest_migration_check; ---> statement-breakpoint -DROP VIEW _manifest_migration_issues; ---> statement-breakpoint ALTER TABLE extensions ADD COLUMN manifest TEXT; --> statement-breakpoint DROP TRIGGER resource_extensions_insert; @@ -208,6 +147,71 @@ CREATE TRIGGER resource_extensions_delete AFTER DELETE ON extensions BEGIN OR (scope='developer' AND subject=OLD.developer_id); END; --> statement-breakpoint +-- MANUAL_MANIFEST_CORRECTIONS +-- Insert reviewed, ID-scoped UPDATE statements here when legacy rows need +-- correction. Accounting remains active; growth quotas are restored below. +-- D1 applies the entire migration atomically, including these corrections. +-- Validate before the canonical backfill. Any remaining issue aborts and +-- rolls back the entire migration, including reviewed corrections above. +CREATE VIEW _manifest_migration_issues AS +WITH published AS ( + SELECT e.*, d.name AS author_name, d.url AS author_url, + CASE WHEN json_valid(e.license) THEN CASE json_type(e.license) + WHEN 'text' THEN json_extract(e.license,'$') + WHEN 'object' THEN CASE + WHEN json_type(e.license,'$.spdx_id') = 'text' THEN json_extract(e.license,'$.spdx_id') + WHEN json_type(e.license,'$.spdx_id') IS NULL OR json_type(e.license,'$.spdx_id') = 'null' + THEN CASE WHEN json_type(e.license,'$.name') = 'text' THEN json_extract(e.license,'$.name') END + END END END AS manifest_license + FROM extensions e JOIN developers d ON d.id = e.developer_id + WHERE e.published_at IS NOT NULL +), release_rows AS ( + SELECT p.id, r.key AS release_index, r.type AS release_type, + CASE WHEN r.type = 'object' THEN r.value ELSE '{}' END AS release + FROM published p, json_each(CASE WHEN json_valid(p.releases) THEN p.releases ELSE '[]' END) r +) +SELECT p.id, NULL AS release_index, problem.value AS issue +FROM published p, json_each(json_array( + CASE WHEN p.type NOT IN ('mod','module','theme','payment-gateway','server-manager','domain-registrar','translation') + THEN 'type: choose a supported manifest type (mod will become module)' END, + CASE WHEN p.manifest_license IS NULL OR length(trim(p.manifest_license)) NOT BETWEEN 1 AND 1000 OR p.manifest_license GLOB '*[^A-Za-z0-9.()+_: |-]*' + THEN 'license: supply a valid manifest license string or legacy spdx_id/name' END, + CASE WHEN length(trim(p.author_name)) = 0 OR length(p.author_name) > 200 + OR (p.author_url IS NOT NULL AND p.author_url != '' AND (length(p.author_url) > 2048 + OR (p.author_url NOT LIKE 'https://%' AND p.author_url NOT LIKE 'http://%') + OR substr(p.author_url,instr(p.author_url,'://')+3,1) IN ('','/','?','#') + OR p.author_url GLOB ('*[' || char(9) || char(10) || char(13) || ' ]*'))) + THEN 'author: correct the developer name/URL before copying package attribution' END, + CASE WHEN CASE WHEN NOT json_valid(p.releases) THEN 1 WHEN json_type(p.releases) != 'array' THEN 1 ELSE json_array_length(p.releases) NOT BETWEEN 1 AND 100 END + THEN 'releases: supply a non-empty JSON array with at most 100 releases' END, + CASE WHEN (SELECT COUNT(*) FROM release_rows r WHERE r.id=p.id AND json_extract(r.release,'$.tag') IN (p.version,'v'||p.version)) != 1 + THEN 'version: identify exactly one current release matching version or v+version' END +)) problem WHERE problem.value IS NOT NULL +UNION ALL +SELECT id, release_index, 'requires: set explicit FOSSBilling and PHP constraints for this release; missing values are not guessed' +FROM release_rows WHERE release_type != 'object' + OR COALESCE(json_type(release,'$.requires'),'') != 'object' + OR COALESCE(json_type(release,'$.requires.fossbilling'),'') != 'text' + OR COALESCE(json_type(release,'$.requires.php'),'') != 'text' + OR COALESCE(length(trim(json_extract(release,'$.requires.fossbilling'))),0) NOT BETWEEN 1 AND 1000 + OR COALESCE(length(trim(json_extract(release,'$.requires.php'))),0) NOT BETWEEN 1 AND 1000 + OR EXISTS (SELECT 1 FROM json_each(CASE WHEN json_type(release,'$.requires') = 'object' THEN json_extract(release,'$.requires') ELSE '{}' END) + WHERE key NOT IN ('fossbilling','php','modules')) + OR (json_type(release,'$.requires.modules') IS NOT NULL AND json_type(release,'$.requires.modules') != 'object') + OR EXISTS (SELECT 1 FROM json_each(CASE WHEN json_type(release,'$.requires.modules') = 'object' THEN json_extract(release,'$.requires.modules') ELSE '{}' END) + WHERE key = '' OR key GLOB '*[^a-z0-9_-]*' OR type != 'text' OR length(trim(value)) NOT BETWEEN 1 AND 1000) +ORDER BY id, release_index, issue; +--> statement-breakpoint +CREATE TABLE _manifest_migration_check ( + invalid_count INTEGER CONSTRAINT manifest_data_requires_manual_correction CHECK (invalid_count = 0) +); +--> statement-breakpoint +INSERT INTO _manifest_migration_check SELECT COUNT(*) FROM _manifest_migration_issues; +--> statement-breakpoint +DROP TABLE _manifest_migration_check; +--> statement-breakpoint +DROP VIEW _manifest_migration_issues; +--> statement-breakpoint UPDATE extensions SET type = CASE WHEN type = 'mod' THEN 'module' ELSE type END, license = json_quote(CASE WHEN json_type(license) = 'object' diff --git a/src/services/extensions/v2/schemas/manifest.ts b/src/services/extensions/v2/schemas/manifest.ts index d1a8f9af..7605348b 100644 --- a/src/services/extensions/v2/schemas/manifest.ts +++ b/src/services/extensions/v2/schemas/manifest.ts @@ -12,12 +12,18 @@ export const MANIFEST_EXTENSION_TYPES = [ "translation" ] as const; +const versionConstraint = z + .string() + .min(1) + .max(1000) + .regex(/\S/, "must contain a version constraint"); + export const ManifestRequirementsSchema = z .object({ - fossbilling: z.string().min(1).max(1000), - php: z.string().min(1).max(1000), + fossbilling: versionConstraint, + php: versionConstraint, modules: z - .record(z.string().regex(/^[a-z0-9_-]+$/), z.string().min(1).max(1000)) + .record(z.string().regex(/^[a-z0-9_-]+$/), versionConstraint) .optional() }) .strict() diff --git a/test/services/extensions/v1/index.test.ts b/test/services/extensions/v1/index.test.ts index 2af7102f..f4feb8a7 100644 --- a/test/services/extensions/v1/index.test.ts +++ b/test/services/extensions/v1/index.test.ts @@ -513,7 +513,7 @@ describe("Extensions API v1", () => { describe("v1 projection failures", () => { beforeAll(applyTestMigrations); - it.each(["not-a-range", ">=2.0.0 <1.0.0"])( + it.each(["not-a-range", ">=2.0.0 <1.0.0", "", " ", "\t\n"])( "flags a requirement with no v1 minimum: %s", async (constraint) => { const db = getExtensionsDb(env.DB_EXTENSIONS); diff --git a/test/services/extensions/v2/manifest.test.ts b/test/services/extensions/v2/manifest.test.ts index a310b855..26e5cc21 100644 --- a/test/services/extensions/v2/manifest.test.ts +++ b/test/services/extensions/v2/manifest.test.ts @@ -179,6 +179,15 @@ describe("manifest v1 publishing", () => { { description: "x".repeat(501) }, { name: "x".repeat(101) }, { requires: { fossbilling: "^1" } }, + { requires: { fossbilling: " ", php: ">=8.3" } }, + { requires: { fossbilling: "^1", php: " " } }, + { + requires: { + fossbilling: "^1", + php: ">=8.3", + modules: { branding: " " } + } + }, { requires: { fossbilling: "^1", diff --git a/test/services/extensions/v2/migrations.test.ts b/test/services/extensions/v2/migrations.test.ts index 7cc35eb7..e6876314 100644 --- a/test/services/extensions/v2/migrations.test.ts +++ b/test/services/extensions/v2/migrations.test.ts @@ -1039,6 +1039,10 @@ it("migrates published manifest metadata without changing reviewed audit bodies" issue: expect.stringContaining("set explicit FOSSBilling and PHP") } ]); + // Existing over-quota collections must survive a structural backfill. + db.exec( + "UPDATE extension_resource_usage SET extensions=101 WHERE scope='developer'" + ); db.exec("BEGIN"); expect(() => db.exec(migration("0029_align_manifest.sql"))).toThrow( /manifest_data_requires_manual_correction/ @@ -1048,19 +1052,34 @@ it("migrates published manifest metadata without changing reviewed audit bodies" expect( db.prepare("SELECT type FROM extensions WHERE id='legacy'").get()?.type ).toBe("mod"); - // Operator supplies actual requirements, not guessed wildcard metadata. - db.prepare( - "UPDATE extensions SET releases=json_set(releases,'$[0].requires',json(?)) WHERE id='legacy'" - ).run(JSON.stringify(currentRequirements)); - expect(db.prepare(preflight).all()).toEqual([]); - // Existing over-quota collections must survive a structural backfill. - db.exec( - "UPDATE extension_resource_usage SET extensions=101 WHERE scope='developer'" + const incompleteCorrection = `UPDATE extensions SET releases=json_set(releases,'$[0].requires',json('{"fossbilling":">=0.8.9"}')) WHERE id='legacy';`; + db.exec("BEGIN"); + expect(() => + db.exec( + migration("0029_align_manifest.sql").replace( + "-- MANUAL_MANIFEST_CORRECTIONS", + `-- MANUAL_MANIFEST_CORRECTIONS\n${incompleteCorrection}` + ) + ) + ).toThrow(/manifest_data_requires_manual_correction/); + db.exec("ROLLBACK"); + expect( + db.prepare("SELECT releases FROM extensions WHERE id='legacy'").get() + ?.releases + ).toBe(releases); + expect(columnNames(db, "extensions")).not.toContain("manifest"); + // The operator inserts an explicit correction into the migration's + // maintenance section; direct growth would still be rejected pre-migration. + const correction = `UPDATE extensions SET releases=json_set(releases,'$[0].requires',json('${JSON.stringify(currentRequirements)}')) WHERE id='legacy';`; + expect(() => db.exec(correction)).toThrow(/extension_resource_quota/); + const preparedMigration = migration("0029_align_manifest.sql").replace( + "-- MANUAL_MANIFEST_CORRECTIONS", + `-- MANUAL_MANIFEST_CORRECTIONS\n${correction}` ); - db.exec("BEGIN"); - db.exec(migration("0029_align_manifest.sql")); + db.exec(preparedMigration); db.exec("COMMIT"); + expect(db.prepare(preflight).all()).toEqual([]); const row = db .prepare( "SELECT type,license,manifest,published_bytes FROM extensions WHERE id='legacy'" From 905d4003654c298d4d8c94e1df2b8727395ae640 Mon Sep 17 00:00:00 2001 From: Adam Daley Date: Wed, 4 Nov 2026 03:44:09 +0000 Subject: [PATCH 5/5] Start cache invalidation regression with cold aliases --- test/services/extensions/v2/moderation-revalidate.test.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/services/extensions/v2/moderation-revalidate.test.ts b/test/services/extensions/v2/moderation-revalidate.test.ts index def2c6b6..fdbad7ee 100644 --- a/test/services/extensions/v2/moderation-revalidate.test.ts +++ b/test/services/extensions/v2/moderation-revalidate.test.ts @@ -1,5 +1,6 @@ import { describe, it, expect, vi, afterEach } from "vitest"; import { env } from "cloudflare:workers"; +import { listCacheKey } from "../../../../src/services/extensions/v1/list-cache"; import { setupExtensionsV2Tests, db, @@ -84,6 +85,10 @@ async function delist(as: string): Promise { describe("CDN cache revalidation on extension mutations", () => { it("invalidates both v1 module cache aliases after a successful mutation", async () => { + // D1 reset does not clear KV. Start cold before priming either alias. + for (const type of ["mod", "module"]) { + await env.CACHE_KV.delete(listCacheKey(type, undefined)); + } await seedModAndExtension(); for (const type of ["mod", "module"]) { const response = await get(`/extensions/v1/list?type=${type}`, {});