diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 62ec2ad5..3123f3e3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,4 +1,4 @@ -name: CI +name: CI/CD on: push: @@ -8,30 +8,174 @@ on: permissions: contents: read +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: - test: + validation: + name: Validation runs-on: ubuntu-latest - if: ${{ (github.event_name == 'pull_request' && github.event.pull_request.base.repo.id != github.event.pull_request.head.repo.id) || github.event_name == 'push' }} + timeout-minutes: 15 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: - name: Checkout Code uses: actions/checkout@0c366fd6a839edf440554fa01a7085ccba70ac98 - - name: Setup Node.js uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f with: node-version: "26.13.0" - cache: "npm" - + cache: npm - name: Install Dependencies + id: dependencies run: npm ci - - - name: Lint Code (ESLint) + - name: Check Formatting + if: ${{ !cancelled() && steps.dependencies.outcome == 'success' }} + run: npm run format:check + - name: Lint Code + if: ${{ !cancelled() && steps.dependencies.outcome == 'success' }} run: npm run lint - - name: Typecheck + if: ${{ !cancelled() && steps.dependencies.outcome == 'success' }} run: npm run typecheck - - name: Run Tests - run: npm run test:all + if: ${{ !cancelled() && steps.dependencies.outcome == 'success' }} + run: npm test + + build: + name: Build API + runs-on: ubuntu-latest + timeout-minutes: 15 + outputs: + wrangler-version: ${{ steps.wrangler-version.outputs.version }} + steps: + - name: Checkout Code + uses: actions/checkout@0c366fd6a839edf440554fa01a7085ccba70ac98 + - name: Setup Node.js + uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f + with: + node-version: "26.13.0" + cache: npm + - name: Install Dependencies + run: npm ci + - name: Read Wrangler Version + id: wrangler-version + run: | + node <<'NODE' + const fs = require('node:fs'); + const { version } = require('./node_modules/wrangler/package.json'); + fs.appendFileSync(process.env.GITHUB_OUTPUT, `version=${version}\n`); + NODE + - name: Build Deployment Artifact + run: npm run build:api + env: + API_PREVIEW_RESOURCES: ${{ vars.API_PREVIEW_RESOURCES }} + - name: Validate Deployment Artifact + run: npm exec -- wrangler deploy --dry-run --config services/api/dist/wrangler.json + - name: Explain Unconfigured Previews + if: github.event_name == 'push' && github.ref_type == 'branch' && github.ref != 'refs/heads/main' && vars.API_PREVIEW_RESOURCES == '' + run: | + echo '::notice::Preview migrations and deployment are skipped until API_PREVIEW_RESOURCES is configured.' + echo '### Preview deployment skipped' >> "$GITHUB_STEP_SUMMARY" + echo 'Configure the API_PREVIEW_RESOURCES repository variable with isolated staging resources to enable branch Previews. Validation and artifact checks still run.' >> "$GITHUB_STEP_SUMMARY" + - name: Upload Deployment Artifact + if: github.event_name == 'push' + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: api-worker-build + overwrite: true + path: services/api/dist/ + if-no-files-found: error + retention-days: 1 + + migrations: + name: Apply API Migrations + needs: [validation, build] + if: github.event_name == 'push' && github.ref_type == 'branch' && (github.ref == 'refs/heads/main' || vars.API_PREVIEW_RESOURCES != '') + runs-on: ubuntu-latest + timeout-minutes: 15 + environment: + name: ${{ github.ref == 'refs/heads/main' && 'production' || 'preview' }} + deployment: false + concurrency: + group: api-migrations-${{ github.ref == 'refs/heads/main' && 'production' || 'preview' }} + cancel-in-progress: false + steps: + - name: Checkout Code + uses: actions/checkout@0c366fd6a839edf440554fa01a7085ccba70ac98 + with: + persist-credentials: false + - name: Setup Node.js + uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f + with: + node-version: "26.13.0" + cache: npm + - name: Install Dependencies + run: npm ci + - name: Download Deployment Artifact + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 + with: + name: api-worker-build + path: services/api/dist/ + - name: Apply Migrations + env: + DEPLOY_TARGET: ${{ github.ref == 'refs/heads/main' && 'production' || 'preview' }} + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + CLOUDFLARE_ACCOUNT_ID: ${{ vars.CLOUDFLARE_ACCOUNT_ID }} + run: | + config=services/api/dist/wrangler.json + if [ "$DEPLOY_TARGET" = preview ]; then + config=services/api/dist/wrangler.preview-migrations.json + if [ ! -f "$config" ]; then + echo '::error::Configure the API_PREVIEW_RESOURCES repository variable before deploying Previews.' + exit 1 + fi + fi + for binding in DB_CENTRAL_ALERTS DB_EXTENSIONS; do + npm exec -- wrangler d1 migrations apply "$binding" --remote --config "$config" + done + + preview: + name: Cloudflare Preview + needs: [validation, build, migrations] + if: ${{ !cancelled() && github.event_name == 'push' && github.ref_type == 'branch' && github.ref != 'refs/heads/main' && vars.API_PREVIEW_RESOURCES != '' }} + permissions: + contents: read + actions: read + pull-requests: write + uses: FOSSBilling/.workflows/.github/workflows/cloudflare-workers.yml@384ecc11e315c77cf3d6abb947e859ce62b38cf5 # main + with: + operation: deploy + mode: preview + build-result: ${{ needs.build.result }} + checks-result: ${{ needs.validation.result == 'success' && needs.migrations.result == 'success' && 'success' || 'failure' }} + pr-comment: true + pr-base-branch: main + worker-name: api + environment: preview + artifact-name: api-worker-build + artifact-path: services/api/dist/ + config-path: services/api/dist/wrangler.json + wrangler-version: ${{ needs.build.outputs.wrangler-version }} + secrets: + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + CLOUDFLARE_ACCOUNT_ID: ${{ vars.CLOUDFLARE_ACCOUNT_ID }} + + deploy: + name: Deploy Production + needs: [validation, build, migrations] + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + uses: FOSSBilling/.workflows/.github/workflows/cloudflare-workers.yml@384ecc11e315c77cf3d6abb947e859ce62b38cf5 # main + with: + operation: deploy + mode: production + worker-name: api + environment: production + artifact-name: api-worker-build + artifact-path: services/api/dist/ + config-path: services/api/dist/wrangler.json + wrangler-version: ${{ needs.build.outputs.wrangler-version }} + secrets: + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + CLOUDFLARE_ACCOUNT_ID: ${{ vars.CLOUDFLARE_ACCOUNT_ID }} diff --git a/.github/workflows/cloudflare-events.yml b/.github/workflows/cloudflare-events.yml new file mode 100644 index 00000000..340f2c9f --- /dev/null +++ b/.github/workflows/cloudflare-events.yml @@ -0,0 +1,23 @@ +name: Cloudflare Events + +on: + workflow_run: + workflows: ["CI/CD"] + types: [completed] + pull_request_target: + branches: [main] + types: [opened, reopened, synchronize] + +permissions: + contents: read + actions: read + pull-requests: write + +jobs: + comment: + uses: FOSSBilling/.workflows/.github/workflows/cloudflare-workers.yml@384ecc11e315c77cf3d6abb947e859ce62b38cf5 # main + with: + operation: sync-pr-comment + build-workflow: ci.yml + pr-base-branch: main + worker-name: api diff --git a/.gitignore b/.gitignore index 481f31b5..06026e2b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,11 +1,12 @@ # prod -dist/ +**/dist/ # dev .vscode/* # deps node_modules/ +.netlify/ .wrangler # env @@ -20,7 +21,7 @@ logs/ npm-debug.log* # test coverage -coverage/ +**/coverage/ # misc .DS_Store diff --git a/.prettierignore b/.prettierignore index a50dd94a..1f011b51 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1,2 +1 @@ -# Ignore workerd generated files -worker-configuration.d.ts \ No newline at end of file +**/worker-configuration.d.ts diff --git a/AGENTS.md b/AGENTS.md index 012befdf..cd1a27d3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,74 +1,15 @@ # Repository Guidelines -## Project Structure & Module Organization - -- `src/app/index.ts` is the worker entrypoint and route wiring for Hono. -- Feature logic lives in `src/services/` and should stay runtime-agnostic. -- Platform interfaces and adapters are in `src/lib/` with Cloudflare and Node implementations under `src/lib/adapters/`. -- Tests mirror the source layout under `test/`, with shared helpers in `test/utils/` and mocks in `test/mocks/`. -- Runtime/config files include `wrangler.jsonc`, `worker-configuration.d.ts`, `tsconfig.json`, `eslint.config.ts`, and `prettier.config.ts`. - -### Service layout - -Small services are flat: `index.ts` + `interfaces.ts` plus narrow helpers (for example `database.ts`, `db/`, `r2.ts`). - -`extensions/v2` is the reference layout for anything larger, and new services should grow into it rather than inventing a third shape: - -- `index.ts` — app assembly only: middleware, route registration, OpenAPI document. Route registration order is load-bearing where static paths must beat parameter paths; those cases carry comments. -- `middleware.ts` — service-specific Hono middleware. -- `routes/` — one module per route group, each exporting `register*Routes(app)`. `routes/errors.ts` maps domain error codes to HTTP status; `routes/app.ts` holds the typed app alias. -- `db/` — `schema.ts`, `migrations/`, one `*Database` class per workflow, plus `errors.ts` (D1 constraint classification) and `batch.ts`. -- `schemas/` — zod/OpenAPI contract split by domain. There is deliberately **no barrel**: import from `schemas/` directly so a module's dependencies are visible. This is why `extensions/v2` has no `interfaces.ts`. -- `github/` — outbound GitHub calls, kept out of the persistence modules. -- `email/` (extensions v2) — provider abstraction and templates. Service-root helpers (`revalidate.ts`, `resource-limits.ts`) stay at the service root. `previews/v1` follows the same split (`routes/`, `github/`, `schemas/`, plus `budget.ts` / `cache.ts`). - -Route modules import `getExtensionsDb`/`getAuth`/`getPlatform` and middleware directly. There is no dependency-injection container; tests drive the real app through `app.request`. - -Each service documents its own contract and operational detail in its own `README.md` (`src/services///README.md`). Keep API behaviour there rather than here or in the root README: this file is for conventions that apply when modifying the code. - -## Build, Test, and Development Commands - -- `npm install`: install dependencies. -- `npm run dev`: start the local Cloudflare Workers dev server. -- `npm run cf-deploy`: apply remote D1 migrations, then deploy via Wrangler. -- `npm run cf-typegen`: regenerate `worker-configuration.d.ts`. -- `npm run db:migrate:all:local` / `db:migrate:all:remote`: apply all D1 migrations locally / remotely. -- `npm run test`: run the default Vitest suite (Workers pool). -- `npm run test:all`: run Workers pool plus Node adapter tests. -- `npm run test:node`: run Node adapter tests via `vitest.node.config.ts`. -- `npm run test:coverage`: run tests with coverage. -- `npm run typecheck`: TypeScript typecheck without emit. -- `npm run lint` / `npm run lint:fix`: lint the codebase (with optional fixes). -- `npm run format` / `npm run format:check`: format or verify formatting with Prettier. - -## Coding Style & Naming Conventions - -- TypeScript (ES modules), 2-space indentation, and Prettier formatting (`trailingComma: "none"`). -- ESLint uses `@typescript-eslint` with Prettier compatibility; keep lint clean before opening PRs. -- Use kebab-case for service folders (for example `central-alerts`), and name tests `*.test.ts`. - -## Testing Guidelines - -- Vitest is the primary test runner; integration tests live in `test/integration/`. -- Prefer colocating tests in the matching `test/` path (for example `src/services/versions/v1` -> `test/services/versions/v1`). -- Run `npm run test:all` before release-related changes to cover both worker and node adapters. - -## Commit & Pull Request Guidelines - -- Recent commits use short, imperative, sentence-case subjects (for example "Update API docs", "Refactor versions service"). -- No ticket prefixes are evident; keep messages focused and under ~72 chars. -- PRs should include a concise summary, testing notes (commands run), and call out config/binding changes in `wrangler.jsonc`. - -## Configuration & Secrets - -- Local secrets go in `.dev.vars` (for example `GITHUB_TOKEN="..."`). -- Bindings, cron, and vars are defined in `wrangler.jsonc` (D1, KV, R2, Durable Objects, rate limiters, service bindings); keep names aligned with `CloudflareBindings`. -- Use Wrangler secrets for production tokens instead of committing them. -- `ASSERTION_SIGNING_SECRET`: HMAC key the extensions v2 API uses to verify short-lived bearer assertions minted by the extensions site (`src/lib/auth/bearer-assertion.ts`). Not sent over the wire — only signs/verifies server-side in each Worker. Add `ASSERTION_SIGNING_SECRET="..."` to `.dev.vars` for local dev; set via `wrangler secret put ASSERTION_SIGNING_SECRET` in production, matching the value configured in the extensions site's Worker. -- `EXTENSIONS_REVALIDATE_SECRET`: bearer token the extensions v2 API sends to the extensions site's `POST /api/revalidate` (over the `EXTENSIONS_FRONTEND` service binding) to purge the site's CDN-cached extension pages after content mutations. Must match the value configured in the extensions site's Worker. Same local/prod setup as `ASSERTION_SIGNING_SECRET`. - -## Cache Revalidation - -Any endpoint that mutates publicly visible content (revision approve/reject, moderator-correct, delist/relist, developer approve, developer profile upsert via `PUT /developers/me`, profile deletion via `DELETE /developers/me`, claim approve/reject, extension withdraw) must call `revalidateExtensions(c)` from `src/services/extensions/v2/revalidate.ts` after a successful write. - -Skipping it does not break correctness — the site's `maxAge`/`stale-while-revalidate` windows bound staleness — but changes then take minutes instead of seconds to appear. Profile deletion keeps its purge even though the `developers` tag is currently a no-op. +This is an npm workspace repository. Deployable components live under `services/`; +reusable libraries live under `libraries/`. Service-specific instructions live in +`services/api/AGENTS.md`. + +- Install dependencies with `npm ci` from the repository root. Keep one root lockfile. +- Services must not import each other's source. Put code shared by actual callers in a private workspace package. +- Declare runtime dependencies in the workspace that uses them. Shared development tools are at the root. +- Preserve the API's existing feature structure inside `services/api/src/services/`. +- Keep provider entry points thin; domain and processing logic should remain portable. +- Local secrets and runtime state belong beside their service (`.dev.vars`, `.wrangler/`). Never commit credentials. +- Run `npm run format:check`, `npm run lint`, `npm run typecheck`, `npm test`, and `npm run build:api` before submitting changes. +- `npm run build:api` is a local dry run. `npm run cf-deploy` applies remote migrations and deploys production. +- Keep commits focused and use short imperative subjects. PRs should explain behaviour, validation and any deployment setup required. diff --git a/README.md b/README.md index 0f4186fb..01a1ca97 100644 --- a/README.md +++ b/README.md @@ -1,165 +1,80 @@ -# FOSSBilling API Worker +# FOSSBilling services -This is the API service that acts as the central hub for FOSSBilling instances. It serves version checks, system-wide alerts, the extensions directory, preview builds, and release stats. +This repository contains separately deployed backend services and the libraries they share. +It uses npm workspaces and one root `package-lock.json`. Install dependencies from this +repository root with `npm ci`; each workspace declares its own dependencies. -Everything is built on [Hono](https://hono.dev), making it lightweight and fast. While we currently deploy this to Cloudflare Workers, the code is designed to be platform-agnostic. +| Workspace | Purpose | Deployment | +| ---------------------------------------------------------------------- | -------------------------------------------------------------------------------------- | ------------------------------------- | +| [`services/api`](services/api/README.md) | HTTP API, authentication, persistence, queues and publication | Cloudflare Workers | +| [`services/extension-packager`](services/extension-packager/README.md) | ZIP validation, manifest replacement and icon preparation | Netlify Function or another Node host | +| [`libraries/extension-packaging`](libraries/extension-packaging) | Shared manifest and processing contracts, ZIP implementation and storage/HTTP adapters | Bundled by its consumers | +| [`libraries/logging`](libraries/logging) | Structured logging and credential redaction used by both services | Bundled by its consumers | -## What it does - -The worker exposes five services: - -- **Versions** (`/versions/v1`) - The source of truth for FOSSBilling updates. It fetches release data from GitHub, caches it for performance, and helps instances decide if they need to update. - -- **Central Alerts** (`/central-alerts/v1`) - Allows the project to push critical notifications to all FOSSBilling installations—useful for security hotfixes or major announcements. - -- **Extensions** (`/extensions/v1`, `/extensions/v2`) - Owns the complete Extensions domain and its `DB_EXTENSIONS` schema. `v2` is authoritative. `v1` is deprecated; its badge URLs retire on **5 February 2027** ([migration and retirement policy](src/services/extensions/v1/README.md)). The separate Extensions site keeps OIDC/session state but accesses this domain through the - generated HTTPS API client; it must not bind or migrate `DB_EXTENSIONS`. - See [`src/services/extensions/v2/README.md`](src/services/extensions/v2/README.md). - -- **Previews** (`/previews/v1`) - Resolves FOSSBilling preview builds — the current main preview and per-PR/per-commit builds produced by FOSSBilling/FOSSBilling's GitHub Actions workflows. Read-only; GitHub Actions and R2 are the sources of truth, not this service. - See [`src/services/previews/v1/README.md`](src/services/previews/v1/README.md). - -- **Stats** (`/stats/v1`) - Aggregates release data through the versions service's shared releases cache into charts and JSON; a cold cache can trigger a GitHub fetch through that shared path. - See [`src/services/stats/v1/README.md`](src/services/stats/v1/README.md). - -## Architecture - -We've structured the app to separate the core logic from the specific runtime environment (Cloudflare, Node, etc.). - -- **Application Logic**: Found in `src/services/versions/v1`, `src/services/central-alerts/v1`, etc. These feature modules don't know they are running on Cloudflare. - Smaller services are a flat `index.ts` + `interfaces.ts`; `src/services/extensions/v2` is the reference layout for larger ones, splitting into `routes/`, `db/`, `schemas/`, and `github/`. - See `AGENTS.md` for what belongs in each. -- **Platform Layer**: Located in `src/lib`. This defines interfaces for things like Cache, Database, and Environment variables. -- **Adapters**: - - `src/lib/adapters/cloudflare`: Real implementations using KV and D1. - - `src/lib/adapters/node`: Reference implementations (useful for testing or alternative deployments). - -Cron and Durable Object code lives with its owning service and is wired in `src/app/index.ts`. - -## APIs - -Each service documents its own endpoints and behaviour: - -| Service | Base path | Docs | -| -------------- | ---------------------------------- | -------------------------------------------------------------------------------------- | -| Versions | `/versions/v1` | [`src/services/versions/v1/README.md`](src/services/versions/v1/README.md) | -| Central Alerts | `/central-alerts/v1` | [`src/services/central-alerts/v1/README.md`](src/services/central-alerts/v1/README.md) | -| Stats | `/stats/v1` | [`src/services/stats/v1/README.md`](src/services/stats/v1/README.md) | -| Extensions | `/extensions/v1`, `/extensions/v2` | [`src/services/extensions/v2/README.md`](src/services/extensions/v2/README.md) | -| Previews | `/previews/v1` | [`src/services/previews/v1/README.md`](src/services/previews/v1/README.md) | - -Extensions v2 and Previews v1 also publish a live OpenAPI document (`/extensions/v2/docs/openapi.json`, `/previews/v1/docs/openapi.json`) and a reference UI (`/extensions/v2/docs`, `/previews/v1/docs`). - -## Configuration - -If you're running this yourself, you'll need a few things set up. - -### Storage - -We use [Cloudflare D1](https://developers.cloudflare.com/d1/) and [KV](https://developers.cloudflare.com/kv/). - -- **D1 Database** (`DB_CENTRAL_ALERTS`): Stores the alert messages. -- **D1 Database** (`DB_EXTENSIONS`): Stores the complete Extensions domain. - Migrations are owned by extensions v2 and applied only from this repository — see [its README](src/services/extensions/v2/README.md#database) for the migration and adoption procedure. -- **KV Namespace** (`CACHE_KV`): Caches GitHub API responses so we don't hit rate limits. -- **KV Namespace** (`AUTH_KV`): Stores the `UPDATE_TOKEN` value for `/versions/v1/update`. -- **R2 Bucket** (`DOWNLOAD_BUCKET`): Shared by versions, previews, and stats for release/preview zips — see [`src/services/previews/v1/README.md`](src/services/previews/v1/README.md) and the comment in `wrangler.jsonc` for which bucket this points at and why. -- **Durable Object** (`PREVIEW_GITHUB_BUDGET`): Token bucket limiting previews GitHub calls. Policy lives in `src/services/previews/v1/budget.ts`. -- **Rate limiters** (`EXTENSION_WRITE_RATE_LIMITER`, `PROFILE_CREATION_RATE_LIMITER`): Edge pacing for extension writes and profile creation. Durable quotas are enforced in D1; see [extensions v2 README](src/services/extensions/v2/README.md#limits-and-budgets). -- **Service binding** (`EXTENSIONS_FRONTEND`): Private binding to the Extensions site worker, used to purge cached extension pages after mutations. -- **Cron** (`0 * * * *`): Hourly extension resource maintenance. See [extensions v2 README](src/services/extensions/v2/README.md#revisions-and-retention). - -### Environment Variables - -Production B2 connection details and MXroute server/username are managed in the -Worker dashboard under **Settings > Variables and Secrets** as runtime variables. -Credentials are stored as secrets. Build environment variables do not configure -the running Worker. `keep_vars: true` preserves dashboard-managed variables on -Wrangler deployments; keep their values out of the committed `vars` map and docs. -Public URLs and deployment controls remain in `wrangler.jsonc`. - -For local development, put operational values and credentials in the ignored -`.dev.vars` file. Document variable names and placeholders, not production values. -Package storage configuration is listed in [PACKAGING.md](src/services/extensions/v2/PACKAGING.md). - -- `GITHUB_TOKEN`: A GitHub Personal Access Token (classic) with public repo read access. -- `ASSERTION_SIGNING_SECRET`: Shared HMAC secret used to verify the short-lived bearer assertions minted by the Extensions site. Configure the same value in both Workers; it is never sent to clients. -- `ASSERTION_SIGNING_SECRET_PREVIOUS`: Optional previous HMAC secret accepted during a signing-key rotation. -- `EXTENSIONS_REVALIDATE_SECRET`: Bearer token the API sends to the Extensions site's `POST /api/revalidate` (over the `EXTENSIONS_FRONTEND` service binding) to purge the site's CDN-cached extension pages after content mutations. Configure the same value in both Workers; it is never sent to clients. -- `EXTENSIONS_V2_EMAIL_PROVIDER`: `mxroute` or `resend` to send moderator notification emails, or unset/`disabled` to skip sending. -- `EXTENSIONS_V2_EMAIL_FROM`: Sender address for notification emails (default `noreply@fossbilling.org`). -- `EXTENSIONS_V2_EMAIL_REPLY_TO`: Optional Reply-To address for a monitored inbox. Omit when unset (no Reply-To header is sent) — notification emails do not invite replies. -- `EXTENSIONS_V2_MXROUTE_SERVER`, `EXTENSIONS_V2_MXROUTE_USERNAME`: Dashboard-managed runtime variables identifying the mailbox for the MXroute SMTP API (`https://smtpapi.mxroute.com/`). Required when `EXTENSIONS_V2_EMAIL_PROVIDER=mxroute`. -- `EXTENSIONS_V2_MXROUTE_PASSWORD`: Mailbox password, stored as a secret. Required when `EXTENSIONS_V2_EMAIL_PROVIDER=mxroute`. -- `EXTENSIONS_V2_RESEND_API_KEY`: API key for Resend. Required when `EXTENSIONS_V2_EMAIL_PROVIDER=resend`. -- `EXTENSIONS_RETENTION_MODE`: `dry-run` (report only) or `compact` (prune old revision bodies). See [extensions v2 README](src/services/extensions/v2/README.md#revisions-and-retention). -- `M2M_ISSUER`, `M2M_JWKS_URL`, `M2M_AUDIENCE`: Issuer, JWKS endpoint, and expected audience for machine-to-machine access tokens minted by the central auth service (`FOSSBilling/auth`). Defaults live in `wrangler.jsonc`; preview environments need them pointed at the issuing preview. See [extensions v2 README](src/services/extensions/v2/README.md#machine-publishing). - -Only extensions v2 consumes these. For the assertion format and the rotation procedure, see [its README](src/services/extensions/v2/README.md#authentication). +Services do not import each other's source. Shared libraries are private npm +workspaces consumed directly from TypeScript source; there is no separate package +publication or library build step. Hosting configuration belongs to its service. ## Development -Get the dependencies installed: +Use Node.js 24 or later. CI tests the workspaces on Node.js 26. Hosting runtime +selection belongs to each service's deployment configuration. Native dependencies +are installed for the build host. -```bash -npm install +```sh +npm ci +npm run dev # API, using services/api/.dev.vars +npm run extension-packager:dev # Node packager, using services/extension-packager/.dev.vars ``` -### Local Setup - -1. Create a `.dev.vars` file for local runtime variables and secrets: - - ```env - GITHUB_TOKEN="your-token" - ASSERTION_SIGNING_SECRET="local-shared-secret" - EXTENSIONS_REVALIDATE_SECRET="local-revalidate-secret" - # Optional while rotating the shared assertion secret. - # ASSERTION_SIGNING_SECRET_PREVIOUS="previous-local-shared-secret" - ``` - -2. Apply migrations to the local D1 databases: - - ```bash - npm run db:migrate:all:local - ``` - -3. (Optional) Store an update token in KV for `/versions/v1/update`: +API migrations and other service-specific commands use npm's workspace option: - ```bash - npx wrangler kv:key put --binding AUTH_KV UPDATE_TOKEN "dev-secret" --local - ``` +```sh +npm run db:migrate:all:local --workspace @fossbilling/api +npm run test:workers --workspace @fossbilling/api +npm test --workspace @fossbilling/extension-packager +``` -4. Spin up the dev server: +See the service READMEs for local storage and authentication configuration. Runtime +secrets and local state stay beside their service and are ignored by Git. - ```bash - npm run dev - ``` +## Checks -You can now hit endpoints at `http://localhost:8787`. +```sh +npm run format:check +npm run lint +npm run typecheck +npm test +npm run build:api # Wrangler dry run; does not deploy or apply remote migrations +``` -### Testing +Each workspace owns its tests and type checking. The root test and type-check +commands run all workspaces. Keep the single root lockfile updated with `npm install` +from the repository root; do not create service-specific lockfiles. -We use Vitest. Unit tests run under the Workers pool; Node adapter tests use a separate config. +## Deployments -```bash -npm run test # workers pool -npm run test:all # workers + node adapters -``` +The services deploy independently from the same repository. Netlify configuration +and setup are documented in the [packager README](services/extension-packager/README.md). -```bash -npm run typecheck -npm run lint -``` +The API uses the same pinned [shared Cloudflare workflow](https://github.com/FOSSBilling/.workflows) +as `fossbilling.org`. [CI/CD](.github/workflows/ci.yml) validates the workspaces and +builds a self-contained Worker artifact, including deployment configuration and SQL. -### Deploy +- Pushes to `main` migrate production D1 and deploy the tested artifact. +- Other branch pushes migrate staging D1 and deploy Previews once `API_PREVIEW_RESOURCES` + is configured; until then, validation and artifact checks run with an explicit Preview skip summary. + Previews have stable branch and + immutable commit URLs recorded on matching PRs. +- PR runs validate without deploying. The [Cloudflare Events workflow](.github/workflows/cloudflare-events.yml) + updates PR comments when a PR opens later or a run finishes, including failed attempts. -```bash -npm run cf-deploy -``` +See the [API deployment setup](services/api/README.md#deployment) for GitHub environments, +staging resource IDs, Preview secrets, migration behaviour and the Workers Builds cutover. +All production and Preview deployments require validation, the build and migrations +to succeed. Previews keep package processing disabled because Cloudflare Previews +cannot consume Queues. Production packaging also remains disabled pending Netlify +runtime verification and the controlled production package test. -This applies remote D1 migrations, then deploys with Wrangler. +`npm run cf-deploy` remains the explicit manual production command; it applies +remote migrations and deploys the API. It does not deploy the packager. diff --git a/eslint.config.ts b/eslint.config.ts index e4d754a2..a425eb45 100644 --- a/eslint.config.ts +++ b/eslint.config.ts @@ -26,8 +26,8 @@ export default defineConfig( } }, { - files: ["**/*.config.ts", "worker-configuration.d.ts"], + files: ["**/*.config.ts", "**/worker-configuration.d.ts"], extends: [tseslint.configs.disableTypeChecked] }, - globalIgnores(["coverage/", ".wrangler/"]) + globalIgnores(["**/coverage/", "**/.wrangler/", "**/dist/", "**/.netlify/"]) ); diff --git a/libraries/extension-packaging/package.json b/libraries/extension-packaging/package.json new file mode 100644 index 00000000..92834d8b --- /dev/null +++ b/libraries/extension-packaging/package.json @@ -0,0 +1,19 @@ +{ + "name": "@fossbilling/extension-packaging", + "private": true, + "version": "0.0.0", + "type": "module", + "license": "AGPL-3.0-only", + "exports": { + "./*": "./src/*.ts" + }, + "scripts": { + "test": "vitest run", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "@zip.js/zip.js": "2.24.0", + "zod": "4.6.7", + "@fossbilling/logging": "*" + } +} diff --git a/src/lib/adapters/b2/package-store.ts b/libraries/extension-packaging/src/adapters/b2.ts similarity index 90% rename from src/lib/adapters/b2/package-store.ts rename to libraries/extension-packaging/src/adapters/b2.ts index eba8a97e..5c49d5a5 100644 --- a/src/lib/adapters/b2/package-store.ts +++ b/libraries/extension-packaging/src/adapters/b2.ts @@ -1,20 +1,16 @@ -import { - PACKAGE_LIMITS, - sha256 -} from "../../../services/extensions/v2/package-builder"; -import { PackageError } from "../../../services/extensions/v2/packages/errors"; -import { - boundedBytes, - type PackageStore -} from "../../../services/extensions/v2/packages/store"; -import { signS3Request } from "./sigv4"; +import { PACKAGE_LIMITS } from "../limits"; +import { sha256 } from "../checksum"; +import { PackageError } from "../errors"; +import { boundedBytes, type PackageStore } from "../store"; +import { signS3Request, type S3RequestInit } from "./sigv4"; // Includes generated manifest and ZIP headers as well as the source contents. const MAX_STORED_BYTES = PACKAGE_LIMITS.expandedBytes + 1024 * 1024; export function b2ObjectStore( getEnv: (name: string) => string | undefined, - kind: "packages" | "assets" + kind: "packages" | "assets", + signal?: AbortSignal ): PackageStore { const get = (suffix: string) => { const value = getEnv(`EXTENSION_${kind.toUpperCase()}_S3_${suffix}`); @@ -73,7 +69,7 @@ export function b2ObjectStore( origin ); } - async function request(url: URL, init: RequestInit) { + async function request(url: URL, init: S3RequestInit) { try { return await fetch( await signS3Request( @@ -83,12 +79,16 @@ export function b2ObjectStore( redirect: "manual", // Read current origin state; public image caching belongs to the CDN. cache: "no-store", - signal: AbortSignal.timeout(15000) + signal: AbortSignal.any([ + ...(signal ? [signal] : []), + AbortSignal.timeout(15000) + ]) }, credentials ) ); } catch { + signal?.throwIfAborted(); throw unavailable("Package storage request failed"); } } @@ -143,6 +143,7 @@ export function b2ObjectStore( try { bytes = await boundedBytes(response.body, MAX_STORED_BYTES); } catch { + signal?.throwIfAborted(); throw unavailable( "Stored package could not be read within its size limit" ); diff --git a/libraries/extension-packaging/src/adapters/http.ts b/libraries/extension-packaging/src/adapters/http.ts new file mode 100644 index 00000000..f4afb4d0 --- /dev/null +++ b/libraries/extension-packaging/src/adapters/http.ts @@ -0,0 +1,96 @@ +import { logError } from "@fossbilling/logging"; +import { + ProcessingResultSchema, + type PackageProcessor, + type ProcessingInput +} from "../processor"; +import { PackageError } from "../errors"; +import { boundedBytes } from "../store"; + +export class HttpPackageProcessor implements PackageProcessor { + private readonly url: URL; + constructor( + url: string, + private readonly secret: string, + local = false + ) { + const target = URL.canParse(url) ? new URL(url) : null; + if ( + !target || + !secret || + target.username || + target.password || + target.hash || + !( + target.protocol === "https:" || + (local && + target.protocol === "http:" && + ["localhost", "127.0.0.1", "[::1]"].includes(target.hostname)) + ) + ) + throw new PackageError( + "PACKAGES_DISABLED", + "Package processor is not configured correctly", + 503 + ); + this.url = target; + } + async process(input: ProcessingInput) { + try { + const response = await fetch( + new Request(this.url, { + method: "POST", + headers: { + Authorization: `Bearer ${this.secret}`, + "Content-Type": "application/json" + }, + body: JSON.stringify(input), + redirect: "manual", + cache: "no-store", + signal: AbortSignal.timeout(50000) + }) + ); + const bytes = await boundedBytes(response.body, 16 * 1024); + const body = JSON.parse(new TextDecoder().decode(bytes)); + if (response.status === 422 && typeof body.error?.message === "string") + throw new PackageError( + body.error.code === "INVALID_ICON" + ? "INVALID_ICON" + : "INVALID_PACKAGE", + body.error.message.slice(0, 1000), + 422 + ); + if (response.status === 503 && body.error?.code === "SOURCE_UNAVAILABLE") + throw new PackageError( + "SOURCE_UNAVAILABLE", + "Source ZIP is unavailable or failed checksum verification", + 503 + ); + if (!response.ok) throw new Error("Package processor request failed"); + const result = ProcessingResultSchema.parse(body.result); + const allowed: readonly string[] = input.icons ?? []; + if ( + result.icons.some((icon) => !allowed.includes(icon.id)) || + new Set(result.icons.map((icon) => icon.id)).size !== + result.icons.length + ) + throw new Error("Package processor returned unexpected output IDs"); + return result; + } catch (error) { + if ( + error instanceof PackageError && + (error.status === 422 || error.code === "SOURCE_UNAVAILABLE") + ) + throw error; + logError("extensions/packages", "Package processor request failed", { + objectId: input.archive, + error: error instanceof Error ? error.message : "Unknown error" + }); + throw new PackageError( + "PACKAGE_UNAVAILABLE", + "Package processor is temporarily unavailable. Please try again.", + 503 + ); + } + } +} diff --git a/src/lib/adapters/b2/sigv4.ts b/libraries/extension-packaging/src/adapters/sigv4.ts similarity index 84% rename from src/lib/adapters/b2/sigv4.ts rename to libraries/extension-packaging/src/adapters/sigv4.ts index d8d0925d..5df63299 100644 --- a/src/lib/adapters/b2/sigv4.ts +++ b/libraries/extension-packaging/src/adapters/sigv4.ts @@ -19,14 +19,14 @@ function toHex(bytes: ArrayBuffer): string { .join(""); } -async function sha256Hex(data: Uint8Array | string): Promise { - const bytes = - typeof data === "string" ? new TextEncoder().encode(data) : data; - return toHex(await crypto.subtle.digest("SHA-256", bytes)); +async function sha256Hex(data: string): Promise { + return toHex( + await crypto.subtle.digest("SHA-256", new TextEncoder().encode(data)) + ); } async function hmac( - key: Uint8Array | ArrayBuffer | string, + key: Uint8Array | ArrayBuffer | string, data: string ): Promise { const bytes = typeof key === "string" ? new TextEncoder().encode(key) : key; @@ -62,19 +62,15 @@ function canonicalQueryString(searchParams: URLSearchParams): string { .join("&"); } -function bodyBytes( - body: BodyInit | null | undefined -): Uint8Array | string | null { - if (body == null) return null; - if (typeof body === "string") return body; - if (body instanceof Uint8Array) return body; - return null; +// PUT callers provide validated bytes and their checksum; other operations have no body. +export interface S3RequestInit extends Omit { + body?: Uint8Array; } /** Sign a request with SigV4 and return a new `Request` carrying the auth headers. */ export async function signS3Request( url: URL | string, - init: RequestInit, + init: S3RequestInit, credentials: S3Credentials, now: Date = new Date() ): Promise { @@ -87,11 +83,11 @@ export async function signS3Request( .replace(/\.\d+Z$/, "Z"); const dateStamp = amzDate.slice(0, 8); - let payloadHash = headers.get("x-amz-content-sha256"); - if (!payloadHash) { - const bytes = bodyBytes(init.body); - payloadHash = bytes == null ? EMPTY_SHA256 : await sha256Hex(bytes); - } + const payloadHash = + headers.get("x-amz-content-sha256") ?? + (init.body == null ? EMPTY_SHA256 : null); + if (!payloadHash) + throw new Error("S3 uploads require an explicit payload checksum"); headers.set("x-amz-date", amzDate); headers.set("x-amz-content-sha256", payloadHash); diff --git a/libraries/extension-packaging/src/checksum.ts b/libraries/extension-packaging/src/checksum.ts new file mode 100644 index 00000000..b550b9de --- /dev/null +++ b/libraries/extension-packaging/src/checksum.ts @@ -0,0 +1,6 @@ +export async function sha256(bytes: Uint8Array) { + const digest = await crypto.subtle.digest("SHA-256", bytes); + return Array.from(new Uint8Array(digest), (byte) => + byte.toString(16).padStart(2, "0") + ).join(""); +} diff --git a/src/services/extensions/v2/packages/errors.ts b/libraries/extension-packaging/src/errors.ts similarity index 100% rename from src/services/extensions/v2/packages/errors.ts rename to libraries/extension-packaging/src/errors.ts diff --git a/src/services/extensions/v2/packages/images.ts b/libraries/extension-packaging/src/images.ts similarity index 100% rename from src/services/extensions/v2/packages/images.ts rename to libraries/extension-packaging/src/images.ts diff --git a/libraries/extension-packaging/src/limits.ts b/libraries/extension-packaging/src/limits.ts new file mode 100644 index 00000000..887260c6 --- /dev/null +++ b/libraries/extension-packaging/src/limits.ts @@ -0,0 +1,8 @@ +// Bounds for uploaded ZIPs and their decompressed contents. +export const PACKAGE_LIMITS = { + sourceBytes: 5 * 1024 * 1024, + expandedBytes: 16 * 1024 * 1024, + entries: 500, + pathLength: 1024, + pathDepth: 32 +} as const; diff --git a/src/services/extensions/v2/schemas/manifest.ts b/libraries/extension-packaging/src/manifest.ts similarity index 81% rename from src/services/extensions/v2/schemas/manifest.ts rename to libraries/extension-packaging/src/manifest.ts index e838dc6c..698dafe3 100644 --- a/src/services/extensions/v2/schemas/manifest.ts +++ b/libraries/extension-packaging/src/manifest.ts @@ -1,5 +1,5 @@ -import { z } from "@hono/zod-openapi"; -import { httpUrl } from "./common"; +import { z } from "zod"; +import { httpUrl } from "./urls"; // Package metadata follows FOSSBilling/modules manifest.schema.json (v1). // Icons use fixed filenames at the package root, outside the manifest. @@ -30,8 +30,7 @@ export const ManifestRequirementsSchema = z .record(z.string().regex(/^[a-z0-9_-]+$/), versionConstraint) .optional() }) - .strict() - .openapi("ManifestRequirements"); + .strict(); export const ManifestSchema = z .object({ @@ -61,17 +60,6 @@ export const ManifestSchema = z homepage_url: httpUrl().optional(), requires: ManifestRequirementsSchema }) - .strict() - .openapi("Manifest"); + .strict(); export type Manifest = z.infer; - -export function manifestMetadata( - content: Pick -) { - return { - manifest_version: content.manifest_version, - author: content.author, - requires: content.requires - }; -} diff --git a/libraries/extension-packaging/src/process.ts b/libraries/extension-packaging/src/process.ts new file mode 100644 index 00000000..11ac68d7 --- /dev/null +++ b/libraries/extension-packaging/src/process.ts @@ -0,0 +1,67 @@ +import { buildExtensionPackage } from "./zip"; +import { PACKAGE_LIMITS } from "./limits"; +import { sha256 } from "./checksum"; +import { PackageError } from "./errors"; +import type { PackageImageProcessor } from "./images"; +import { boundedBytes, type PackageStore } from "./store"; +import { + ProcessingResultSchema, + type ProcessingInput, + type ProcessingResult +} from "./processor"; + +export async function processPackage( + input: ProcessingInput, + store: PackageStore, + images: PackageImageProcessor, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted(); + const source = await boundedBytes( + await store.get(input.source.id, input.source.sha256), + PACKAGE_LIMITS.sourceBytes + ); + if (!source.length || (await sha256(source)) !== input.source.sha256) + throw new PackageError( + "SOURCE_UNAVAILABLE", + "Source ZIP is unavailable or failed checksum verification", + 503 + ); + let built; + try { + built = await buildExtensionPackage(source, input.manifest, signal); + } catch (error) { + signal?.throwIfAborted(); + throw new PackageError( + "INVALID_PACKAGE", + error instanceof Error + ? error.message + : "Source ZIP failed package validation", + 422 + ); + } + const variants = + input.icons && built.icon + ? await images.prepare(built.icon, built.iconName === "icon.svg") + : []; + signal?.throwIfAborted(); + await store.put(input.archive, built.bytes, built.packageSha256); + const icons: ProcessingResult["icons"] = []; + for (const [index, variant] of variants.entries()) { + signal?.throwIfAborted(); + const id = input.icons![index]; + const checksum = await sha256(variant.bytes); + await store.put(id, variant.bytes, checksum); + icons.push({ + id, + format: variant.format, + sha256: checksum, + bytes: variant.bytes.length + }); + } + signal?.throwIfAborted(); + return ProcessingResultSchema.parse({ + archive: { bytes: built.bytes.length, sha256: built.packageSha256 }, + icons + }); +} diff --git a/libraries/extension-packaging/src/processor.ts b/libraries/extension-packaging/src/processor.ts new file mode 100644 index 00000000..cd262be1 --- /dev/null +++ b/libraries/extension-packaging/src/processor.ts @@ -0,0 +1,57 @@ +import { z } from "zod"; +import { ManifestSchema } from "./manifest"; +import { PACKAGE_LIMITS } from "./limits"; + +export const MAX_PACKAGE_OBJECT_BYTES = + PACKAGE_LIMITS.expandedBytes + 1024 * 1024; +// Matches the sanitizer's maximum prepared SVG output. +export const MAX_PACKAGE_ICON_BYTES = 4 * 1024 * 1024; +const checksum = z.string().regex(/^[a-f0-9]{64}$/); +const output = z + .object({ + bytes: z.number().int().positive().max(MAX_PACKAGE_OBJECT_BYTES), + sha256: checksum + }) + .strict(); +export const ProcessingInputSchema = z + .object({ + manifest: ManifestSchema, + source: z.object({ id: z.uuid(), sha256: checksum }).strict(), + archive: z.uuid(), + icons: z.tuple([z.uuid(), z.uuid()]).optional() + }) + .strict() + .refine((value) => { + const keys = [value.source.id, value.archive, ...(value.icons ?? [])]; + return new Set(keys).size === keys.length; + }, "Processing object IDs must be distinct"); +export const ProcessingResultSchema = z + .object({ + archive: output, + icons: z + .array( + output + .extend({ + bytes: output.shape.bytes.max(MAX_PACKAGE_ICON_BYTES), + id: z.uuid(), + format: z.enum(["svg", "avif", "png"]) + }) + .strict() + ) + .max(2) + }) + .strict() + .refine((value) => { + const formats = value.icons + .map((icon) => icon.format) + .sort() + .join(","); + return formats === "" || formats === "svg" || formats === "avif,png"; + }, "Invalid prepared icon variants"); +export type ProcessingInput = z.infer; +export type ProcessingResult = z.infer; + +// Storage keys and manifests are provider-neutral; the runner never needs a database binding. +export interface PackageProcessor { + process(input: ProcessingInput): Promise; +} diff --git a/src/services/extensions/v2/packages/store.ts b/libraries/extension-packaging/src/store.ts similarity index 100% rename from src/services/extensions/v2/packages/store.ts rename to libraries/extension-packaging/src/store.ts diff --git a/libraries/extension-packaging/src/urls.ts b/libraries/extension-packaging/src/urls.ts new file mode 100644 index 00000000..84f08b62 --- /dev/null +++ b/libraries/extension-packaging/src/urls.ts @@ -0,0 +1,12 @@ +import { z } from "zod"; + +// Restricts to http(s) — z.url() alone accepts any scheme, +// including javascript:/data:, which is unsafe for fields a consumer may +// render as a link or image src. +export const httpUrl = () => + z + .url() + .max(2048) + .refine((value) => /^https?:\/\//i.test(value), { + message: "must use http or https" + }); diff --git a/src/services/extensions/v2/package-builder.ts b/libraries/extension-packaging/src/zip.ts similarity index 90% rename from src/services/extensions/v2/package-builder.ts rename to libraries/extension-packaging/src/zip.ts index 59aa497d..5c1a6e3e 100644 --- a/src/services/extensions/v2/package-builder.ts +++ b/libraries/extension-packaging/src/zip.ts @@ -1,19 +1,12 @@ +import { PACKAGE_LIMITS } from "./limits"; +import { sha256 } from "./checksum"; import { Uint8ArrayReader, Uint8ArrayWriter, ZipReader, ZipWriter } from "@zip.js/zip.js"; -import { ManifestSchema } from "./schemas/manifest"; - -// Bounds for uploaded ZIPs and their decompressed contents. -export const PACKAGE_LIMITS = { - sourceBytes: 5 * 1024 * 1024, - expandedBytes: 16 * 1024 * 1024, - entries: 500, - pathLength: 1024, - pathDepth: 32 -} as const; +import { ManifestSchema } from "./manifest"; const zipOptions = { useWebWorkers: false, @@ -23,13 +16,6 @@ const zipOptions = { checkOverlappingEntry: true }; -export async function sha256(bytes: Uint8Array) { - const digest = await crypto.subtle.digest("SHA-256", bytes); - return Array.from(new Uint8Array(digest), (byte) => - byte.toString(16).padStart(2, "0") - ).join(""); -} - function validatePath(filename: string, directory: boolean) { const path = directory ? filename.replace(/\/$/, "") : filename; if (path.length > PACKAGE_LIMITS.pathLength) @@ -58,13 +44,17 @@ function validatePath(filename: string, directory: boolean) { /** Validate source files and build a bounded archive with directory metadata. */ export async function buildExtensionPackage( source: Uint8Array, - manifestInput: unknown + manifestInput: unknown, + signal?: AbortSignal ) { const manifest = ManifestSchema.parse(manifestInput); if (source.byteLength > PACKAGE_LIMITS.sourceBytes) { throw new Error("Source ZIP exceeds size limit"); } - const reader = new ZipReader(new Uint8ArrayReader(source), zipOptions); + const reader = new ZipReader(new Uint8ArrayReader(source), { + ...zipOptions, + signal + }); const files: { path: string; bytes: Uint8Array; @@ -186,6 +176,7 @@ export async function buildExtensionPackage( files.sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0)); const writer = new ZipWriter(new Uint8ArrayWriter(), { ...zipOptions, + signal, level: 6, lastModDate: new Date("1980-01-01T00:00:00Z"), extendedTimestamp: false diff --git a/test/services/extensions/v2/b2-store.node.test.ts b/libraries/extension-packaging/test/b2-store.test.ts similarity index 87% rename from test/services/extensions/v2/b2-store.node.test.ts rename to libraries/extension-packaging/test/b2-store.test.ts index 3dafd7f7..df9bc82f 100644 --- a/test/services/extensions/v2/b2-store.node.test.ts +++ b/libraries/extension-packaging/test/b2-store.test.ts @@ -1,9 +1,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; -import { b2ObjectStore } from "../../../../src/lib/adapters/b2/package-store"; -import { - sha256, - PACKAGE_LIMITS -} from "../../../../src/services/extensions/v2/package-builder"; +import { b2ObjectStore } from "../src/adapters/b2"; +import { sha256 } from "../src/checksum"; +import { PACKAGE_LIMITS } from "../src/limits"; const objectId = "134cf413-b5ab-4f57-b2ad-520020af880d"; const config: Record = { @@ -22,6 +20,39 @@ const store = (overrides: Record = {}) => afterEach(() => vi.unstubAllGlobals()); describe("B2 object storage", () => { + it("preserves a caller deadline while reading from B2", async () => { + const controller = new AbortController(); + let started!: () => void; + const fetching = new Promise((resolve) => { + started = resolve; + }); + vi.stubGlobal( + "fetch", + (request: Request) => + new Promise((_resolve, reject) => { + request.signal.addEventListener( + "abort", + () => reject(request.signal.reason), + { once: true } + ); + started(); + }) + ); + const storage = b2ObjectStore( + (name) => config[name.replace("EXTENSION_PACKAGES_S3_", "")], + "packages", + controller.signal + ); + const deadline = new DOMException( + "Processor deadline exceeded", + "TimeoutError" + ); + const read = expect(storage.get(objectId)).rejects.toBe(deadline); + await fetching; + controller.abort(deadline); + await read; + }); + it("signs ZIP writes and verifies downloaded bytes", async () => { const bytes = new Uint8Array(new TextEncoder().encode("synthetic ZIP")); const checksum = await sha256(bytes); diff --git a/test/services/extensions/v2/sigv4.node.test.ts b/libraries/extension-packaging/test/sigv4.test.ts similarity index 98% rename from test/services/extensions/v2/sigv4.node.test.ts rename to libraries/extension-packaging/test/sigv4.test.ts index 701e26de..8870367e 100644 --- a/test/services/extensions/v2/sigv4.node.test.ts +++ b/libraries/extension-packaging/test/sigv4.test.ts @@ -1,6 +1,6 @@ import { createHash, createHmac } from "node:crypto"; import { describe, expect, it } from "vitest"; -import { signS3Request } from "../../../../src/lib/adapters/b2/sigv4"; +import { signS3Request } from "../src/adapters/sigv4"; const credentials = { accessKeyId: "test-key", diff --git a/test/services/extensions/v2/package-builder.test.ts b/libraries/extension-packaging/test/zip.test.ts similarity index 95% rename from test/services/extensions/v2/package-builder.test.ts rename to libraries/extension-packaging/test/zip.test.ts index e3dc87a8..d0e3519b 100644 --- a/test/services/extensions/v2/package-builder.test.ts +++ b/libraries/extension-packaging/test/zip.test.ts @@ -7,11 +7,9 @@ import { ZipReader, ZipWriter } from "@zip.js/zip.js"; -import { - buildExtensionPackage, - PACKAGE_LIMITS, - sha256 -} from "../../../../src/services/extensions/v2/package-builder"; +import { buildExtensionPackage } from "../src/zip"; +import { PACKAGE_LIMITS } from "../src/limits"; +import { sha256 } from "../src/checksum"; const manifest = { manifest_version: 1, @@ -50,6 +48,17 @@ async function contents(bytes: Uint8Array) { } describe("directory package builder", () => { + it("stops ZIP processing when its caller aborts", async () => { + const source = await zip([["Service.php", " { for (const path of [ "a/".repeat(20000) + "file.php", diff --git a/libraries/extension-packaging/tsconfig.json b/libraries/extension-packaging/tsconfig.json new file mode 100644 index 00000000..e8b02ca4 --- /dev/null +++ b/libraries/extension-packaging/tsconfig.json @@ -0,0 +1,4 @@ +{ + "extends": "../../tsconfig.json", + "include": ["src/**/*.ts", "test/**/*.ts"] +} diff --git a/libraries/logging/package.json b/libraries/logging/package.json new file mode 100644 index 00000000..182d9480 --- /dev/null +++ b/libraries/logging/package.json @@ -0,0 +1,12 @@ +{ + "name": "@fossbilling/logging", + "private": true, + "version": "0.0.0", + "type": "module", + "license": "AGPL-3.0-only", + "exports": "./src/logger.ts", + "scripts": { + "test": "vitest run", + "typecheck": "tsc --noEmit" + } +} diff --git a/src/lib/logger.ts b/libraries/logging/src/logger.ts similarity index 100% rename from src/lib/logger.ts rename to libraries/logging/src/logger.ts diff --git a/test/lib/logger.test.ts b/libraries/logging/test/logger.test.ts similarity index 98% rename from test/lib/logger.test.ts rename to libraries/logging/test/logger.test.ts index 495e8117..6a714021 100644 --- a/test/lib/logger.test.ts +++ b/libraries/logging/test/logger.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; -import { logError, logWarn, logInfo } from "../../src/lib/logger"; +import { logError, logWarn, logInfo } from "../src/logger"; // console[level] receives one structured object ({ service, message, // context? }) - the shape Workers Logs serializes to filterable JSON. diff --git a/libraries/logging/tsconfig.json b/libraries/logging/tsconfig.json new file mode 100644 index 00000000..e8b02ca4 --- /dev/null +++ b/libraries/logging/tsconfig.json @@ -0,0 +1,4 @@ +{ + "extends": "../../tsconfig.json", + "include": ["src/**/*.ts", "test/**/*.ts"] +} diff --git a/package-lock.json b/package-lock.json index 0a47263b..8f563ec7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,45 +1,48 @@ { - "name": "api", + "name": "@fossbilling/services", "lockfileVersion": 3, "requires": true, "packages": { "": { - "name": "api", + "name": "@fossbilling/services", "license": "AGPL-3.0-only", - "dependencies": { - "@hono/zod-openapi": "1.6.3", - "@octokit/graphql": "9.0.5", - "@octokit/request": "10.0.16", - "@scalar/hono-api-reference": "0.12.18", - "@zeropress/svg-hush": "0.1.1", - "@zip.js/zip.js": "2.24.0", - "badge-maker": "6.0.0", - "better-auth": "^1.7.14", - "drizzle-orm": "0.45.4", - "hono": "4.13.17", - "semver": "7.8.6", - "uuid": "14.0.3", - "zod": "4.6.7" - }, + "workspaces": [ + "services/*", + "libraries/*" + ], "devDependencies": { - "@cloudflare/vitest-plugin": "1.6.1", "@eslint/js": "10.0.1", "@types/node": "26.6.6", - "@types/semver": "7.8.0", "@typescript-eslint/eslint-plugin": "8.72.1", "@typescript-eslint/parser": "8.72.1", "@vitest/coverage-istanbul": "5.1.0", - "drizzle-kit": "0.31.11", "eslint": "10.14.0", "eslint-config-prettier": "10.1.8", "jiti": "2.7.0", "prettier": "3.9.10", "typescript": "6.0.3", "typescript-eslint": "8.72.1", - "vitest": "5.1.0", - "wrangler": "4.158.0" + "vitest": "5.1.0" + }, + "engines": { + "node": ">=24" } }, + "libraries/extension-packaging": { + "name": "@fossbilling/extension-packaging", + "version": "0.0.0", + "license": "AGPL-3.0-only", + "dependencies": { + "@fossbilling/logging": "*", + "@zip.js/zip.js": "2.24.0", + "zod": "4.6.7" + } + }, + "libraries/logging": { + "name": "@fossbilling/logging", + "version": "0.0.0", + "license": "AGPL-3.0-only" + }, "node_modules/@asteasolutions/zod-to-openapi": { "version": "9.1.0", "resolved": "https://registry.npmjs.org/@asteasolutions/zod-to-openapi/-/zod-to-openapi-9.1.0.tgz", @@ -1454,6 +1457,22 @@ "node": "^20.19.0 || ^22.13.0 || >=24" } }, + "node_modules/@fossbilling/api": { + "resolved": "services/api", + "link": true + }, + "node_modules/@fossbilling/extension-packager": { + "resolved": "services/extension-packager", + "link": true + }, + "node_modules/@fossbilling/extension-packaging": { + "resolved": "libraries/extension-packaging", + "link": true + }, + "node_modules/@fossbilling/logging": { + "resolved": "libraries/logging", + "link": true + }, "node_modules/@hono/zod-openapi": { "version": "1.6.3", "resolved": "https://registry.npmjs.org/@hono/zod-openapi/-/zod-openapi-1.6.3.tgz", @@ -1552,7 +1571,6 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/@img/colour/-/colour-1.1.0.tgz", "integrity": "sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ==", - "dev": true, "license": "MIT", "engines": { "node": ">=18" @@ -1565,7 +1583,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "Apache-2.0", "optional": true, "os": [ @@ -1588,7 +1605,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "Apache-2.0", "optional": true, "os": [ @@ -1608,7 +1624,6 @@ "version": "0.35.5", "resolved": "https://registry.npmjs.org/@img/sharp-freebsd-wasm32/-/sharp-freebsd-wasm32-0.35.5.tgz", "integrity": "sha512-Y/z91nEZ4uIBX5X3nfTovjU9lHNKFYbL2lpHCLVNmXQK03VIZvXBBt0KxbPGp2SdGSF+2mQU4e+hQaWOt86iAw==", - "dev": true, "license": "Apache-2.0", "optional": true, "os": [ @@ -1631,7 +1646,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1648,7 +1662,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1665,7 +1678,6 @@ "cpu": [ "arm" ], - "dev": true, "libc": [ "glibc" ], @@ -1685,7 +1697,6 @@ "cpu": [ "arm64" ], - "dev": true, "libc": [ "glibc" ], @@ -1705,7 +1716,6 @@ "cpu": [ "ppc64" ], - "dev": true, "libc": [ "glibc" ], @@ -1725,7 +1735,6 @@ "cpu": [ "riscv64" ], - "dev": true, "libc": [ "glibc" ], @@ -1745,7 +1754,6 @@ "cpu": [ "s390x" ], - "dev": true, "libc": [ "glibc" ], @@ -1765,7 +1773,6 @@ "cpu": [ "x64" ], - "dev": true, "libc": [ "glibc" ], @@ -1785,7 +1792,6 @@ "cpu": [ "arm64" ], - "dev": true, "libc": [ "musl" ], @@ -1805,7 +1811,6 @@ "cpu": [ "x64" ], - "dev": true, "libc": [ "musl" ], @@ -1825,7 +1830,6 @@ "cpu": [ "arm" ], - "dev": true, "libc": [ "glibc" ], @@ -1851,7 +1855,6 @@ "cpu": [ "arm64" ], - "dev": true, "libc": [ "glibc" ], @@ -1877,7 +1880,6 @@ "cpu": [ "ppc64" ], - "dev": true, "libc": [ "glibc" ], @@ -1903,7 +1905,6 @@ "cpu": [ "riscv64" ], - "dev": true, "libc": [ "glibc" ], @@ -1929,7 +1930,6 @@ "cpu": [ "s390x" ], - "dev": true, "libc": [ "glibc" ], @@ -1955,7 +1955,6 @@ "cpu": [ "x64" ], - "dev": true, "libc": [ "glibc" ], @@ -1981,7 +1980,6 @@ "cpu": [ "arm64" ], - "dev": true, "libc": [ "musl" ], @@ -2007,7 +2005,6 @@ "cpu": [ "x64" ], - "dev": true, "libc": [ "musl" ], @@ -2030,7 +2027,6 @@ "version": "0.35.5", "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.35.5.tgz", "integrity": "sha512-Ptsga1su4tQx+LLF1ECS9U6nz5kmrXKo6XVbtR48Ke3ZRxxgaWBu7IDtEe1quo8hiupwm6WFqxVlXaSf7IINGQ==", - "dev": true, "license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT", "optional": true, "dependencies": { @@ -2050,7 +2046,6 @@ "cpu": [ "wasm32" ], - "dev": true, "license": "Apache-2.0", "optional": true, "dependencies": { @@ -2070,7 +2065,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "Apache-2.0 AND LGPL-3.0-or-later", "optional": true, "os": [ @@ -2090,7 +2084,6 @@ "cpu": [ "ia32" ], - "dev": true, "license": "Apache-2.0 AND LGPL-3.0-or-later", "optional": true, "os": [ @@ -2110,7 +2103,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "Apache-2.0 AND LGPL-3.0-or-later", "optional": true, "os": [ @@ -3577,7 +3569,6 @@ "version": "2.1.2", "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", - "devOptional": true, "license": "Apache-2.0", "engines": { "node": ">=8" @@ -5529,7 +5520,6 @@ "version": "0.35.5", "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.35.5.tgz", "integrity": "sha512-Ywn4OnzGukp7CDMrp08RQ50YKmuwG47brZgIVPTvBaaAfQlRlygrRqSrxdCiL9M+LlzLBiJ68IR1QqvzHyjC7g==", - "dev": true, "license": "Apache-2.0", "dependencies": { "@img/colour": "^1.1.0", @@ -6654,6 +6644,48 @@ "funding": { "url": "https://github.com/sponsors/colinhacks" } + }, + "services/api": { + "name": "@fossbilling/api", + "version": "0.0.0", + "license": "AGPL-3.0-only", + "dependencies": { + "@fossbilling/extension-packaging": "*", + "@fossbilling/logging": "*", + "@hono/zod-openapi": "1.6.3", + "@octokit/graphql": "9.0.5", + "@octokit/request": "10.0.16", + "@scalar/hono-api-reference": "0.12.18", + "badge-maker": "6.0.0", + "better-auth": "^1.7.14", + "drizzle-orm": "0.45.4", + "hono": "4.13.17", + "semver": "7.8.6", + "uuid": "14.0.3", + "zod": "4.6.7" + }, + "devDependencies": { + "@cloudflare/vitest-plugin": "1.6.1", + "@types/semver": "7.8.0", + "@zip.js/zip.js": "2.24.0", + "drizzle-kit": "0.31.11", + "wrangler": "4.158.0" + } + }, + "services/extension-packager": { + "name": "@fossbilling/extension-packager", + "version": "0.0.0", + "license": "AGPL-3.0-only", + "dependencies": { + "@fossbilling/extension-packaging": "*", + "@fossbilling/logging": "*", + "@zeropress/svg-hush": "0.1.1", + "sharp": "0.35.5" + }, + "devDependencies": { + "@zip.js/zip.js": "2.24.0", + "tsx": "4.23.15" + } } } } diff --git a/package.json b/package.json index 572de1d3..160a8b5a 100644 --- a/package.json +++ b/package.json @@ -1,61 +1,41 @@ { - "name": "api", + "name": "@fossbilling/services", + "private": true, "license": "AGPL-3.0-only", "type": "module", + "workspaces": [ + "services/*", + "libraries/*" + ], + "engines": { + "node": ">=24" + }, "scripts": { - "cf-typegen": "wrangler types --env-interface CloudflareBindings", - "cf-deploy": "npm run db:migrate:all:remote && wrangler deploy --minify", - "dev": "wrangler dev", + "dev": "npm run dev --workspace @fossbilling/api", + "extension-packager:dev": "npm run dev --workspace @fossbilling/extension-packager", + "cf-typegen": "npm run cf-typegen --workspace @fossbilling/api", + "cf-deploy": "npm run cf-deploy --workspace @fossbilling/api", + "build:api": "npm run build --workspace @fossbilling/api", + "test": "npm test --workspaces", + "typecheck": "tsc --noEmit && npm run typecheck --workspaces", + "lint": "eslint .", "format": "prettier --write .", "format:check": "prettier --check .", - "db:generate:extensions": "drizzle-kit generate --config drizzle.extensions.config.ts", - "db:generate:central-alerts": "drizzle-kit generate --config drizzle.central-alerts.config.ts", - "db:migrate:extensions-v2:local": "wrangler d1 migrations apply DB_EXTENSIONS --local", - "db:migrate:extensions-v2:remote": "wrangler d1 migrations apply DB_EXTENSIONS --remote", - "db:migrate:central-alerts:local": "wrangler d1 migrations apply DB_CENTRAL_ALERTS --local", - "db:migrate:central-alerts:remote": "wrangler d1 migrations apply DB_CENTRAL_ALERTS --remote", - "db:migrate:all:local": "npm run db:migrate:extensions-v2:local && npm run db:migrate:central-alerts:local", - "db:migrate:all:remote": "npm run db:migrate:extensions-v2:remote && npm run db:migrate:central-alerts:remote", - "lint": "eslint", - "lint:fix": "eslint --fix", - "test": "vitest run", - "test:node": "vitest run --config vitest.node.config.ts", - "test:all": "npm run test && npm run test:node", - "test:coverage": "vitest run --coverage", - "typecheck": "tsc --noEmit" - }, - "dependencies": { - "@hono/zod-openapi": "1.6.3", - "@octokit/graphql": "9.0.5", - "@octokit/request": "10.0.16", - "@scalar/hono-api-reference": "0.12.18", - "@zeropress/svg-hush": "0.1.1", - "@zip.js/zip.js": "2.24.0", - "badge-maker": "6.0.0", - "better-auth": "^1.7.14", - "drizzle-orm": "0.45.4", - "hono": "4.13.17", - "semver": "7.8.6", - "uuid": "14.0.3", - "zod": "4.6.7" + "cf-preview": "npm run cf-preview --workspace @fossbilling/api" }, "devDependencies": { - "@cloudflare/vitest-plugin": "1.6.1", "@eslint/js": "10.0.1", "@types/node": "26.6.6", - "@types/semver": "7.8.0", "@typescript-eslint/eslint-plugin": "8.72.1", "@typescript-eslint/parser": "8.72.1", "@vitest/coverage-istanbul": "5.1.0", - "drizzle-kit": "0.31.11", "eslint": "10.14.0", "eslint-config-prettier": "10.1.8", "jiti": "2.7.0", "prettier": "3.9.10", "typescript": "6.0.3", "typescript-eslint": "8.72.1", - "vitest": "5.1.0", - "wrangler": "4.158.0" + "vitest": "5.1.0" }, "overrides": { "@esbuild-kit/core-utils": { diff --git a/services/api/AGENTS.md b/services/api/AGENTS.md new file mode 100644 index 00000000..0f118e90 --- /dev/null +++ b/services/api/AGENTS.md @@ -0,0 +1,78 @@ +# Repository Guidelines + +## Project Structure & Module Organization + +- `src/app/index.ts` is the worker entrypoint and route wiring for Hono. +- Feature logic lives in `src/services/` and should stay runtime-agnostic. +- Platform interfaces and adapters are in `src/lib/` with Cloudflare and Node implementations under `src/lib/adapters/`. +- Tests mirror the source layout under `test/`, with shared helpers in `test/utils/` and mocks in `test/mocks/`. +- Service configuration includes `wrangler.jsonc`, `worker-configuration.d.ts`, `tsconfig.json`, and the Drizzle/Vitest configs. ESLint and Prettier configuration live at the repository root. + +### Service layout + +Small services are flat: `index.ts` + `interfaces.ts` plus narrow helpers (for example `database.ts`, `db/`, `r2.ts`). + +`extensions/v2` is the reference layout for anything larger, and new services should grow into it rather than inventing a third shape: + +- `index.ts` — app assembly only: middleware, route registration, OpenAPI document. Route registration order is load-bearing where static paths must beat parameter paths; those cases carry comments. +- `middleware.ts` — service-specific Hono middleware. +- `routes/` — one module per route group, each exporting `register*Routes(app)`. `routes/errors.ts` maps domain error codes to HTTP status; `routes/app.ts` holds the typed app alias. +- `db/` — `schema.ts`, `migrations/`, one `*Database` class per workflow, plus `errors.ts` (D1 constraint classification) and `batch.ts`. +- `schemas/` — zod/OpenAPI contract split by domain. There is deliberately **no barrel**: import from `schemas/` directly so a module's dependencies are visible. This is why `extensions/v2` has no `interfaces.ts`. +- `github/` — outbound GitHub calls, kept out of the persistence modules. +- `email/` (extensions v2) — provider abstraction and templates. Service-root helpers (`revalidate.ts`, `resource-limits.ts`) stay at the service root. `previews/v1` follows the same split (`routes/`, `github/`, `schemas/`, plus `budget.ts` / `cache.ts`). + +Use `getPlatform(c)` for cache access, service requests, rate-limit checks and background tasks. Keep binding conversion in platform adapters and inject background scheduling from the host entry point. + +Route modules import `getExtensionsDb`/`getAuth`/`getPlatform` and middleware directly. There is no dependency-injection container; tests drive the real app through `app.request`. + +Each service documents its own contract and operational detail in its own `README.md` (`src/services///README.md`). Keep API behaviour there rather than here or in the root README: this file is for conventions that apply when modifying the code. + +## Build, Test, and Development Commands + +- Run `npm ci` at the repository root to install workspace dependencies. +- `npm run dev`: start the local Cloudflare Workers dev server. +- `npm run cf-deploy`: apply remote D1 migrations, then deploy via Wrangler. +- `npm run cf-typegen`: regenerate `worker-configuration.d.ts`. +- `npm run build`: create a validated Worker artifact with deployment configuration and migrations. +- `npm run cf-preview`: deploy the prepared artifact as a Cloudflare Preview after staging migrations. +- `npm run db:migrate:all:local` / `db:migrate:all:remote`: apply all D1 migrations locally / remotely. +- `npm test`: run Workers and Node adapter tests for the API. +- `npm run test:workers`: run only the Workers pool. +- `npm run test:node`: run Node adapter tests via `vitest.node.config.ts`. +- `npm run test:coverage`: run tests with coverage. +- `npm run typecheck`: TypeScript typecheck without emit. +- Run `npm run lint` from the repository root. +- Run `npm run format` / `npm run format:check` from the repository root. + +## Coding Style & Naming Conventions + +- TypeScript (ES modules), 2-space indentation, and Prettier formatting (`trailingComma: "none"`). +- ESLint uses `@typescript-eslint` with Prettier compatibility; keep lint clean before opening PRs. +- Use kebab-case for service folders (for example `central-alerts`), and name tests `*.test.ts`. + +## Testing Guidelines + +- Vitest is the primary test runner; integration tests live in `test/integration/`. +- Prefer colocating tests in the matching `test/` path (for example `src/services/versions/v1` -> `test/services/versions/v1`). +- Run `npm test` at the repository root before release-related changes to cover all workspaces. + +## Commit & Pull Request Guidelines + +- Recent commits use short, imperative, sentence-case subjects (for example "Update API docs", "Refactor versions service"). +- No ticket prefixes are evident; keep messages focused and under ~72 chars. +- PRs should include a concise summary, testing notes (commands run), and call out config/binding changes in `wrangler.jsonc`. + +## Configuration & Secrets + +- Local secrets go in `.dev.vars` (for example `GITHUB_TOKEN="..."`). +- Bindings, cron, and vars are defined in `wrangler.jsonc` (D1, KV, R2, Durable Objects, rate limiters, service bindings); keep names aligned with `CloudflareBindings`. +- Use Wrangler secrets for production tokens instead of committing them. +- `ASSERTION_SIGNING_SECRET`: HMAC key the extensions v2 API uses to verify short-lived bearer assertions minted by the extensions site (`src/lib/auth/bearer-assertion.ts`). Not sent over the wire — only signs/verifies server-side in each Worker. Add `ASSERTION_SIGNING_SECRET="..."` to `.dev.vars` for local dev; set via `wrangler secret put ASSERTION_SIGNING_SECRET` in production, matching the value configured in the extensions site's Worker. +- `EXTENSIONS_REVALIDATE_SECRET`: bearer token the extensions v2 API sends to the extensions site's `POST /api/revalidate` (over the `EXTENSIONS_FRONTEND` service binding) to purge the site's CDN-cached extension pages after content mutations. Must match the value configured in the extensions site's Worker. Same local/prod setup as `ASSERTION_SIGNING_SECRET`. + +## Cache Revalidation + +Any endpoint that mutates publicly visible content (revision approve/reject, moderator-correct, delist/relist, developer approve, developer profile upsert via `PUT /developers/me`, profile deletion via `DELETE /developers/me`, claim approve/reject, extension withdraw) must call `revalidateExtensions(c)` from `src/services/extensions/v2/revalidate.ts` after a successful write. + +Skipping it does not break correctness — the site's `maxAge`/`stale-while-revalidate` windows bound staleness — but changes then take minutes instead of seconds to appear. Profile deletion keeps its purge even though the `developers` tag is currently a no-op. diff --git a/services/api/README.md b/services/api/README.md new file mode 100644 index 00000000..133df719 --- /dev/null +++ b/services/api/README.md @@ -0,0 +1,243 @@ +# FOSSBilling API Worker + +This service is an npm workspace. Run `npm ci` from the [repository root](../../README.md). +The commands below run from `services/api` unless stated otherwise. + +This is the API service that acts as the central hub for FOSSBilling instances. It serves version checks, system-wide alerts, the extensions directory, preview builds, and release stats. + +The HTTP layer uses [Hono](https://hono.dev) and deploys to Cloudflare Workers. Shared packaging code and several platform capabilities use provider-neutral interfaces; database and specialised storage integrations still use Cloudflare bindings. + +## What it does + +The worker exposes five services: + +- **Versions** (`/versions/v1`) + The source of truth for FOSSBilling updates. It fetches release data from GitHub, caches it for performance, and helps instances decide if they need to update. + +- **Central Alerts** (`/central-alerts/v1`) + Allows the project to push critical notifications to all FOSSBilling installations—useful for security hotfixes or major announcements. + +- **Extensions** (`/extensions/v1`, `/extensions/v2`) + Owns the complete Extensions domain and its `DB_EXTENSIONS` schema. `v2` is authoritative. `v1` is deprecated; its badge URLs retire on **5 February 2027** ([migration and retirement policy](src/services/extensions/v1/README.md)). The separate Extensions site keeps OIDC/session state but accesses this domain through the + generated HTTPS API client; it must not bind or migrate `DB_EXTENSIONS`. + See [`src/services/extensions/v2/README.md`](src/services/extensions/v2/README.md). + +- **Previews** (`/previews/v1`) + Resolves FOSSBilling preview builds — the current main preview and per-PR/per-commit builds produced by FOSSBilling/FOSSBilling's GitHub Actions workflows. Read-only; GitHub Actions and R2 are the sources of truth, not this service. + See [`src/services/previews/v1/README.md`](src/services/previews/v1/README.md). + +- **Stats** (`/stats/v1`) + Aggregates release data through the versions service's shared releases cache into charts and JSON; a cold cache can trigger a GitHub fetch through that shared path. + See [`src/services/stats/v1/README.md`](src/services/stats/v1/README.md). + +## Architecture + +Feature modules live under `src/services/`. Smaller services are flat; larger ones, +such as `extensions/v2`, split into `routes/`, `db/`, `schemas/` and `github/`. + +`src/lib/context.ts` exposes caches, queues, environment values, service requests, +rate-limit checks and background-task scheduling. Cloudflare adapters translate +bindings into these capabilities. The Worker entry point supplies the request's +`waitUntil` callback; another host can supply its own task scheduling. Application +helpers use ordinary `Request`/`Response` values rather than Cloudflare `Fetcher`s. + +Database code uses Drizzle's D1 driver and some features access R2 or Durable Objects +directly. Moving the API to another host would still require replacing those +integrations. The Node adapters currently provide caches and environment access, +not a complete alternative API runtime. + +Package processing uses the shared library's storage, processor and image interfaces. +B2 storage configuration lives in `extensions/v2/packages/store.ts`; Cloudflare Queue +delivery and acknowledgement stay in `src/lib/adapters/cloudflare/packaging-jobs.ts`. +The separately hosted packager does not import API source or Cloudflare bindings. + +## APIs + +Each service documents its own endpoints and behaviour: + +| Service | Base path | Docs | +| -------------- | ---------------------------------- | -------------------------------------------------------------------------------------- | +| Versions | `/versions/v1` | [`src/services/versions/v1/README.md`](src/services/versions/v1/README.md) | +| Central Alerts | `/central-alerts/v1` | [`src/services/central-alerts/v1/README.md`](src/services/central-alerts/v1/README.md) | +| Stats | `/stats/v1` | [`src/services/stats/v1/README.md`](src/services/stats/v1/README.md) | +| Extensions | `/extensions/v1`, `/extensions/v2` | [`src/services/extensions/v2/README.md`](src/services/extensions/v2/README.md) | +| Previews | `/previews/v1` | [`src/services/previews/v1/README.md`](src/services/previews/v1/README.md) | + +Extensions v2 and Previews v1 also publish a live OpenAPI document (`/extensions/v2/docs/openapi.json`, `/previews/v1/docs/openapi.json`) and a reference UI (`/extensions/v2/docs`, `/previews/v1/docs`). + +## Configuration + +If you're running this yourself, you'll need a few things set up. + +### Storage + +We use [Cloudflare D1](https://developers.cloudflare.com/d1/) and [KV](https://developers.cloudflare.com/kv/). + +- **D1 Database** (`DB_CENTRAL_ALERTS`): Stores the alert messages. +- **D1 Database** (`DB_EXTENSIONS`): Stores the complete Extensions domain. + Migrations are owned by extensions v2 and applied only from this repository — see [its README](src/services/extensions/v2/README.md#database) for the migration and adoption procedure. +- **KV Namespace** (`CACHE_KV`): Caches GitHub API responses so we don't hit rate limits. +- **KV Namespace** (`AUTH_KV`): Stores the `UPDATE_TOKEN` value for `/versions/v1/update`. +- **R2 Bucket** (`DOWNLOAD_BUCKET`): Shared by versions, previews, and stats for release/preview zips — see [`src/services/previews/v1/README.md`](src/services/previews/v1/README.md) and the comment in `wrangler.jsonc` for which bucket this points at and why. +- **Durable Object** (`PREVIEW_GITHUB_BUDGET`): Token bucket limiting previews GitHub calls. Policy lives in `src/services/previews/v1/budget.ts`. +- **Rate limiters** (`EXTENSION_WRITE_RATE_LIMITER`, `PROFILE_CREATION_RATE_LIMITER`): Edge pacing for extension writes and profile creation. Durable quotas are enforced in D1; see [extensions v2 README](src/services/extensions/v2/README.md#limits-and-budgets). +- **Service binding** (`EXTENSIONS_FRONTEND`): Private binding to the Extensions site worker, used to purge cached extension pages after mutations. +- **Cron** (`0 * * * *`): Hourly extension resource maintenance. See [extensions v2 README](src/services/extensions/v2/README.md#revisions-and-retention). + +### Environment Variables + +Production B2 connection details and MXroute server/username are managed in the +Worker dashboard under **Settings > Variables and Secrets** as runtime variables. +Credentials are stored as secrets. Build environment variables do not configure +the running Worker. `keep_vars: true` preserves dashboard-managed variables on +Wrangler deployments; keep their values out of the committed `vars` map and docs. +Public URLs and deployment controls remain in `wrangler.jsonc`. + +For local development, put operational values and credentials in the ignored +`.dev.vars` file. Document variable names and placeholders, not production values. +Package storage configuration is listed in [PACKAGING.md](src/services/extensions/v2/PACKAGING.md). + +- `GITHUB_TOKEN`: A GitHub Personal Access Token (classic) with public repo read access. +- `ASSERTION_SIGNING_SECRET`: Shared HMAC secret used to verify the short-lived bearer assertions minted by the Extensions site. Configure the same value in both Workers; it is never sent to clients. +- `ASSERTION_SIGNING_SECRET_PREVIOUS`: Optional previous HMAC secret accepted during a signing-key rotation. +- `EXTENSIONS_REVALIDATE_SECRET`: Bearer token the API sends to the Extensions site's `POST /api/revalidate` (over the `EXTENSIONS_FRONTEND` service binding) to purge the site's CDN-cached extension pages after content mutations. Configure the same value in both Workers; it is never sent to clients. +- `EXTENSIONS_V2_EMAIL_PROVIDER`: `mxroute` or `resend` to send moderator notification emails, or unset/`disabled` to skip sending. +- `EXTENSIONS_V2_EMAIL_FROM`: Sender address for notification emails (default `noreply@fossbilling.org`). +- `EXTENSIONS_V2_EMAIL_REPLY_TO`: Optional Reply-To address for a monitored inbox. Omit when unset (no Reply-To header is sent) — notification emails do not invite replies. +- `EXTENSIONS_V2_MXROUTE_SERVER`, `EXTENSIONS_V2_MXROUTE_USERNAME`: Dashboard-managed runtime variables identifying the mailbox for the MXroute SMTP API (`https://smtpapi.mxroute.com/`). Required when `EXTENSIONS_V2_EMAIL_PROVIDER=mxroute`. +- `EXTENSIONS_V2_MXROUTE_PASSWORD`: Mailbox password, stored as a secret. Required when `EXTENSIONS_V2_EMAIL_PROVIDER=mxroute`. +- `EXTENSIONS_V2_RESEND_API_KEY`: API key for Resend. Required when `EXTENSIONS_V2_EMAIL_PROVIDER=resend`. +- `EXTENSIONS_RETENTION_MODE`: `dry-run` (report only) or `compact` (prune old revision bodies). See [extensions v2 README](src/services/extensions/v2/README.md#revisions-and-retention). +- `M2M_ISSUER`, `M2M_JWKS_URL`, `M2M_AUDIENCE`: Issuer, JWKS endpoint, and expected audience for machine-to-machine access tokens minted by the central auth service (`FOSSBilling/auth`). Defaults live in `wrangler.jsonc`; preview environments need them pointed at the issuing preview. See [extensions v2 README](src/services/extensions/v2/README.md#machine-publishing). + +The extension-specific settings apply to extensions v2; Versions, Previews and +Stats also use `GITHUB_TOKEN`. For the assertion format and the rotation procedure, see [its README](src/services/extensions/v2/README.md#authentication). + +## Development + +### Local Setup + +1. Create a `.dev.vars` file for local runtime variables and secrets: + + ```env + GITHUB_TOKEN="your-token" + ASSERTION_SIGNING_SECRET="local-shared-secret" + EXTENSIONS_REVALIDATE_SECRET="local-revalidate-secret" + # Optional while rotating the shared assertion secret. + # ASSERTION_SIGNING_SECRET_PREVIOUS="previous-local-shared-secret" + ``` + +2. Apply migrations to the local D1 databases: + + ```bash + npm run db:migrate:all:local + ``` + +3. (Optional) Store an update token in KV for `/versions/v1/update`: + + ```bash + npx wrangler kv:key put --binding AUTH_KV UPDATE_TOKEN "dev-secret" --local + ``` + +4. Spin up the dev server: + + ```bash + npm run dev + ``` + +You can now hit endpoints at `http://localhost:8787`. + +### Testing + +We use Vitest. Unit tests run under the Workers pool; Node adapter tests use a separate config. + +```bash +npm run test:workers # workers pool +npm test # workers + node adapters +``` + +```bash +npm run typecheck +npm run lint --prefix ../.. +``` + +### Deployment + +[CI/CD](../../.github/workflows/ci.yml) uses the pinned shared Cloudflare workflow +from [FOSSBilling/.workflows](https://github.com/FOSSBilling/.workflows), matching +`fossbilling.org`'s Preview URLs, deployment summaries and PR comment history. + +Validation and artifact builds run on branch pushes and PRs. A push can deploy only +after workspace validation, the build and D1 migrations succeed. `main` targets +production; other branches target Previews once `API_PREVIEW_RESOURCES` is configured. +Until then, branch builds report that Preview migrations and deployment are skipped; +validation and artifact checks still run. PR runs, including forks, never receive +deployment credentials or apply remote migrations. + +The artifact contains the compiled Worker, `wrangler.json`, and SQL migrations. +Deployment does not rebuild the Worker. Production migrations use that artifact's +production bindings. Preview migrations use its generated +`wrangler.preview-migrations.json`, derived from the same bindings used by the Preview. +Migration jobs serialize access to the shared staging databases. + +#### Setup before switching deployment + +1. Configure GitHub `production` and `preview` environments, each with its own + `CLOUDFLARE_API_TOKEN` secret. Set `CLOUDFLARE_ACCOUNT_ID` as a repository variable + shared by both deployment targets. Restrict the production environment to `main`. + Deployment tokens need Workers deployment and + D1 migration permissions; Preview credentials and resources must be non-production. +2. Provision staging D1 databases, KV namespaces, a download bucket and a separately + deployed staging Extensions frontend. Set the repository variable `API_PREVIEW_RESOURCES` + to their identifiers, using the following JSON shape: + + ```json + { + "DB_EXTENSIONS": "", + "DB_CENTRAL_ALERTS": "", + "AUTH_KV": "", + "CACHE_KV": "", + "DOWNLOAD_BUCKET": "", + "EXTENSIONS_FRONTEND": "" + } + ``` + + The build refuses production resource IDs and duplicated IDs. These are operational + identifiers, not credentials. The variable is optional for local production bundle + checks. When absent, CI skips Preview migrations and deployment; a malformed or + production resource configuration still fails the build. Previews share these + staging resources; they do not automatically receive separate D1/KV/R2 data. + +3. Configure Preview runtime secrets in the API Worker's **Previews Base** settings. + Use a separate assertion-signing secret, revalidation secret and read-only GitHub + token. Match the revalidation secret with the staging frontend. Set M2M issuer, + JWKS and audience variables there when using a separate test issuer. Operational + production variables and secrets remain dashboard-managed with `keep_vars: true`. +4. Configure required checks for the new validation/build job names, then disable the + API's Cloudflare Workers Builds Git integration to avoid duplicate deployments. + +The workflow compiles the branch's Preview settings into the artifact. Each Preview +receives isolated Durable Object state; the data bindings above are shared staging +resources. A frontend service binding calls that Worker's normal deployment, so use a +staging Worker name rather than the production Extensions worker. + +Cloudflare Previews cannot consume Queues or receive Cron Triggers. Their configuration +therefore disables packaging and notification email, and omits Queue bindings. Full +packaging/consumer testing needs a separately deployed staging Worker and Queue, or the +controlled production test. See [Cloudflare's Preview resource behaviour](https://developers.cloudflare.com/workers/previews/resources/). + +The [Cloudflare Events workflow](../../.github/workflows/cloudflare-events.yml) runs the +trusted shared comment handler on workflow completion and PR events. It reads deployment +metadata without checking out or executing PR code. Renovate tracks the pinned shared +workflow references in both files. + +#### Local deployment commands + +From the repository root, `npm run build:api` validates the bundle and prepares the +artifact without remote writes. To prepare a Preview artifact, supply +`API_PREVIEW_RESOURCES` in the environment when building. Apply its staging migrations +before running `npm run cf-preview`; CI performs those steps automatically. + +`npm run cf-deploy` remains a deliberate manual production command. It applies remote +migrations and deploys through Wrangler. Neither command deploys the separate +[extension packager](../extension-packager/README.md). Production packaging remains disabled. diff --git a/drizzle.central-alerts.config.ts b/services/api/drizzle.central-alerts.config.ts similarity index 100% rename from drizzle.central-alerts.config.ts rename to services/api/drizzle.central-alerts.config.ts diff --git a/drizzle.extensions.config.ts b/services/api/drizzle.extensions.config.ts similarity index 100% rename from drizzle.extensions.config.ts rename to services/api/drizzle.extensions.config.ts diff --git a/services/api/package.json b/services/api/package.json new file mode 100644 index 00000000..a50a79e0 --- /dev/null +++ b/services/api/package.json @@ -0,0 +1,49 @@ +{ + "name": "@fossbilling/api", + "private": true, + "version": "0.0.0", + "type": "module", + "license": "AGPL-3.0-only", + "scripts": { + "cf-typegen": "wrangler types --env-interface CloudflareBindings", + "cf-deploy": "npm run db:migrate:all:remote && wrangler deploy --minify", + "dev": "wrangler dev", + "db:generate:extensions": "drizzle-kit generate --config drizzle.extensions.config.ts", + "db:generate:central-alerts": "drizzle-kit generate --config drizzle.central-alerts.config.ts", + "db:migrate:extensions-v2:local": "wrangler d1 migrations apply DB_EXTENSIONS --local", + "db:migrate:extensions-v2:remote": "wrangler d1 migrations apply DB_EXTENSIONS --remote", + "db:migrate:central-alerts:local": "wrangler d1 migrations apply DB_CENTRAL_ALERTS --local", + "db:migrate:central-alerts:remote": "wrangler d1 migrations apply DB_CENTRAL_ALERTS --remote", + "db:migrate:all:local": "npm run db:migrate:extensions-v2:local && npm run db:migrate:central-alerts:local", + "db:migrate:all:remote": "npm run db:migrate:extensions-v2:remote && npm run db:migrate:central-alerts:remote", + "test": "npm run test:workers && npm run test:node", + "test:node": "vitest run --config vitest.node.config.ts", + "test:coverage": "vitest run --coverage", + "typecheck": "tsc --noEmit", + "test:workers": "vitest run", + "build": "wrangler deploy --dry-run --outdir dist --minify && node scripts/prepare-deployment.ts", + "cf-preview": "wrangler preview --config dist/wrangler.json" + }, + "dependencies": { + "@hono/zod-openapi": "1.6.3", + "@octokit/graphql": "9.0.5", + "@octokit/request": "10.0.16", + "@scalar/hono-api-reference": "0.12.18", + "badge-maker": "6.0.0", + "better-auth": "^1.7.14", + "drizzle-orm": "0.45.4", + "hono": "4.13.17", + "semver": "7.8.6", + "uuid": "14.0.3", + "zod": "4.6.7", + "@fossbilling/extension-packaging": "*", + "@fossbilling/logging": "*" + }, + "devDependencies": { + "@cloudflare/vitest-plugin": "1.6.1", + "@types/semver": "7.8.0", + "drizzle-kit": "0.31.11", + "wrangler": "4.158.0", + "@zip.js/zip.js": "2.24.0" + } +} diff --git a/services/api/scripts/prepare-deployment.ts b/services/api/scripts/prepare-deployment.ts new file mode 100644 index 00000000..da9851ff --- /dev/null +++ b/services/api/scripts/prepare-deployment.ts @@ -0,0 +1,115 @@ +import { + copyFileSync, + mkdirSync, + readdirSync, + rmSync, + writeFileSync +} from "node:fs"; +import { join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { experimental_readRawConfig, type Unstable_RawConfig } from "wrangler"; +import { z } from "zod"; + +const PreviewResourcesSchema = z.strictObject({ + DB_EXTENSIONS: z.uuid().toLowerCase(), + DB_CENTRAL_ALERTS: z.uuid().toLowerCase(), + AUTH_KV: z.string().regex(/^[a-f0-9]{32}$/), + CACHE_KV: z.string().regex(/^[a-f0-9]{32}$/), + DOWNLOAD_BUCKET: z.string().min(1), + EXTENSIONS_FRONTEND: z.string().min(1) +}); + +// Keep the bundle, runtime bindings and migration targets in one deployable artifact. +export function prepareDeployment(root: string, previewResources?: string) { + const config: Unstable_RawConfig = experimental_readRawConfig({ + config: join(root, "wrangler.jsonc") + }).rawConfig; + const output = join(root, "dist"); + const resources = previewResources + ? PreviewResourcesSchema.parse(JSON.parse(previewResources)) + : undefined; + if (resources) { + const production = new Set([ + ...(config.d1_databases ?? []).map((value) => value.database_id), + ...(config.kv_namespaces ?? []).map((value) => value.id), + ...(config.r2_buckets ?? []).map((value) => value.bucket_name), + ...(config.services ?? []).map((value) => value.service) + ]); + const values = Object.values(resources); + if ( + values.some((value) => production.has(value)) || + resources.DB_EXTENSIONS === resources.DB_CENTRAL_ALERTS || + resources.AUTH_KV === resources.CACHE_KV + ) + throw new Error( + "API_PREVIEW_RESOURCES must use distinct non-production resources" + ); + } + mkdirSync(output, { recursive: true }); + rmSync(join(output, "migrations"), { recursive: true, force: true }); + rmSync(join(output, "wrangler.preview-migrations.json"), { force: true }); + config.d1_databases = (config.d1_databases ?? []).map((database) => { + if (!database.migrations_dir) + throw new Error(`Missing migrations_dir for ${database.binding}`); + const migrations = `migrations/${database.binding}`; + mkdirSync(join(output, migrations), { recursive: true }); + for (const filename of readdirSync( + join(root, database.migrations_dir) + ).filter((value) => value.endsWith(".sql"))) + copyFileSync( + join(root, database.migrations_dir, filename), + join(output, migrations, filename) + ); + return { ...database, migrations_dir: migrations }; + }); + if (resources) { + const databaseIds: Record = { + DB_EXTENSIONS: resources.DB_EXTENSIONS, + DB_CENTRAL_ALERTS: resources.DB_CENTRAL_ALERTS + }; + const stagingDatabases = (config.d1_databases ?? []).map((database) => { + const id = databaseIds[database.binding]; + if (!id) + throw new Error( + `No Preview database configured for ${database.binding}` + ); + return { ...database, database_id: id, database_name: undefined }; + }); + config.previews = { + ...config.previews, + d1_databases: stagingDatabases, + kv_namespaces: [ + { binding: "AUTH_KV", id: resources.AUTH_KV }, + { binding: "CACHE_KV", id: resources.CACHE_KV } + ], + r2_buckets: [ + { binding: "DOWNLOAD_BUCKET", bucket_name: resources.DOWNLOAD_BUCKET } + ], + services: [ + { + binding: "EXTENSIONS_FRONTEND", + service: resources.EXTENSIONS_FRONTEND + } + ] + }; + writeFileSync( + join(output, "wrangler.preview-migrations.json"), + JSON.stringify({ d1_databases: stagingDatabases }, null, 2) + "\n" + ); + } + Reflect.deleteProperty(config, "$schema"); + config.main = "index.js"; + config.no_bundle = true; + writeFileSync( + join(output, "wrangler.json"), + JSON.stringify(config, null, 2) + "\n" + ); +} + +if ( + process.argv[1] && + resolve(process.argv[1]) === fileURLToPath(import.meta.url) +) { + prepareDeployment(process.cwd(), process.env.API_PREVIEW_RESOURCES); + console.log("Prepared API deployment configuration and migrations."); +} diff --git a/src/app/index.ts b/services/api/src/app/index.ts similarity index 76% rename from src/app/index.ts rename to services/api/src/app/index.ts index 0f775643..9df2d587 100644 --- a/src/app/index.ts +++ b/services/api/src/app/index.ts @@ -1,7 +1,7 @@ import { - consumePackageJobs, - recoverPackageJobs -} from "../lib/adapters/cloudflare/package-jobs"; + consumePackagingJobs, + recoverPackagingJobs +} from "../lib/adapters/cloudflare/packaging-jobs"; import { getExtensionsDb } from "../lib/db"; import { maintainExtensionResources, @@ -9,21 +9,23 @@ import { } from "../services/extensions/v2/db/resource-maintenance"; import { Hono } from "hono"; import { HTTPException } from "hono/http-exception"; -import centralAlertsV1 from "../services/central-alerts/v1"; -import extensionsV1 from "../services/extensions/v1"; -import extensionsV2 from "../services/extensions/v2"; -import previewsV1 from "../services/previews/v1"; -import versionsV1 from "../services/versions/v1"; -import statsV1 from "../services/stats/v1"; +import centralAlertsV1 from "../services/central-alerts/v1/index"; +import extensionsV1 from "../services/extensions/v1/index"; +import extensionsV2 from "../services/extensions/v2/index"; +import previewsV1 from "../services/previews/v1/index"; +import versionsV1 from "../services/versions/v1/index"; +import statsV1 from "../services/stats/v1/index"; import { platformMiddleware } from "../lib/middleware"; -import { createCloudflareBindings } from "../lib/adapters/cloudflare"; -import { logError } from "../lib/logger"; +import { createCloudflareBindings } from "../lib/adapters/cloudflare/index"; +import { logError } from "@fossbilling/logging"; const app = new Hono<{ Bindings: CloudflareBindings }>(); app.use("*", async (c, next) => { const bindings = createCloudflareBindings(c.env); - const middleware = platformMiddleware(bindings); + const middleware = platformMiddleware(bindings, (promise) => + c.executionCtx.waitUntil(promise) + ); return middleware(c, next); }); @@ -79,7 +81,7 @@ app.all("/*", (c) => { }); export default { - queue: consumePackageJobs, + queue: consumePackagingJobs, fetch: app.fetch, request: app.request.bind(app), scheduled: async ( @@ -87,9 +89,9 @@ export default { env: CloudflareBindings ): Promise => { try { - await recoverPackageJobs(env); + await recoverPackagingJobs(env); } catch (error) { - logError("extensions/packages", "Package job recovery failed", { + logError("extensions/packages", "Packaging job recovery failed", { error: error instanceof Error ? error.message : String(error) }); } diff --git a/src/lib/adapters/cloudflare/cache.ts b/services/api/src/lib/adapters/cloudflare/cache.ts similarity index 100% rename from src/lib/adapters/cloudflare/cache.ts rename to services/api/src/lib/adapters/cloudflare/cache.ts diff --git a/src/lib/adapters/cloudflare/environment.ts b/services/api/src/lib/adapters/cloudflare/environment.ts similarity index 100% rename from src/lib/adapters/cloudflare/environment.ts rename to services/api/src/lib/adapters/cloudflare/environment.ts diff --git a/src/lib/adapters/cloudflare/index.ts b/services/api/src/lib/adapters/cloudflare/index.ts similarity index 79% rename from src/lib/adapters/cloudflare/index.ts rename to services/api/src/lib/adapters/cloudflare/index.ts index 89f80c7b..cdac7578 100644 --- a/src/lib/adapters/cloudflare/index.ts +++ b/services/api/src/lib/adapters/cloudflare/index.ts @@ -26,6 +26,15 @@ export function createCloudflareBindings( ) } : {}, + services: { + EXTENSIONS_FRONTEND: (request) => env.EXTENSIONS_FRONTEND.fetch(request) + }, + rateLimiters: { + EXTENSION_WRITE_RATE_LIMITER: async (key) => + (await env.EXTENSION_WRITE_RATE_LIMITER.limit({ key })).success, + PROFILE_CREATION_RATE_LIMITER: async (key) => + (await env.PROFILE_CREATION_RATE_LIMITER.limit({ key })).success + }, caches: { CACHE_KV: new CloudflareKVAdapter(env.CACHE_KV), AUTH_KV: new CloudflareKVAdapter(env.AUTH_KV) diff --git a/src/lib/adapters/cloudflare/package-assets.ts b/services/api/src/lib/adapters/cloudflare/package-assets.ts similarity index 92% rename from src/lib/adapters/cloudflare/package-assets.ts rename to services/api/src/lib/adapters/cloudflare/package-assets.ts index 82b2e283..0acb9494 100644 --- a/src/lib/adapters/cloudflare/package-assets.ts +++ b/services/api/src/lib/adapters/cloudflare/package-assets.ts @@ -1,9 +1,9 @@ import { PackageBuildDatabase } from "../../../services/extensions/v2/db/package-builds"; -import { logError } from "../../logger"; +import { logError } from "@fossbilling/logging"; import { revalidateExtensions } from "../../../services/extensions/v2/revalidate"; import type { Context } from "hono"; import { getPlatform } from "../../middleware"; -import { PackageError } from "../../../services/extensions/v2/packages/errors"; +import { PackageError } from "@fossbilling/extension-packaging/errors"; // Publishing changes the listing URL only; icons are already stored in B2. async function updatePublishedIconUrl( @@ -48,7 +48,7 @@ export function schedulePublishedIconUpdate( extensionId: string ) { if (!getPlatform(c).getEnv("EXTENSION_ASSETS_S3_ENDPOINT")) return; - c.executionCtx.waitUntil( + getPlatform(c).waitUntil( updatePublishedIconUrl(c, extensionId) .then((changed) => { if (changed) revalidateExtensions(c); diff --git a/src/lib/adapters/cloudflare/package-jobs.ts b/services/api/src/lib/adapters/cloudflare/packaging-jobs.ts similarity index 59% rename from src/lib/adapters/cloudflare/package-jobs.ts rename to services/api/src/lib/adapters/cloudflare/packaging-jobs.ts index 8e50757e..ed03821b 100644 --- a/src/lib/adapters/cloudflare/package-jobs.ts +++ b/services/api/src/lib/adapters/cloudflare/packaging-jobs.ts @@ -1,25 +1,25 @@ -import { logError } from "../../logger"; +import { logError } from "@fossbilling/logging"; import { createCloudflareBindings } from "./index"; -import { PackageJobsDatabase } from "../../../services/extensions/v2/db/package-jobs"; +import { PackagingJobsDatabase } from "../../../services/extensions/v2/db/packaging-jobs"; import { PackageBuildDatabase } from "../../../services/extensions/v2/db/package-builds"; import { prepareQueuedPackage, type PackagePreparation } from "../../../services/extensions/v2/packages/prepare"; import { - processPackageJob, - dispatchPackageJob + processPackagingJob, + dispatchPackagingJob } from "../../../services/extensions/v2/packages/jobs"; -import { PackageError } from "../../../services/extensions/v2/packages/errors"; -import { configuredPackageStore } from "./package-store"; -import { CloudflarePackageImages } from "./package-images"; +import { PackageError } from "@fossbilling/extension-packaging/errors"; +import { configuredPackageStore } from "../../../services/extensions/v2/packages/store"; +import { configuredPackageProcessor } from "../../../services/extensions/v2/packages/processor"; export function packagePreparation( env: CloudflareBindings ): PackagePreparation { const bindings = createCloudflareBindings(env); const getEnv = (name: string) => bindings.environment.get(name); - const store = configuredPackageStore(env, getEnv); + const store = configuredPackageStore(getEnv); const baseUrl = getEnv("EXTENSION_PACKAGES_BASE_URL"); if (!store || !baseUrl) throw new PackageError( @@ -30,16 +30,16 @@ export function packagePreparation( return { repository: new PackageBuildDatabase(env.DB_EXTENSIONS), store, - images: new CloudflarePackageImages(env.PACKAGE_IMAGES), + processor: configuredPackageProcessor(getEnv), baseUrl }; } -export async function consumePackageJobs( +export async function consumePackagingJobs( batch: MessageBatch<{ jobId: string }>, env: CloudflareBindings ) { - const jobs = new PackageJobsDatabase(env.DB_EXTENSIONS); + const jobs = new PackagingJobsDatabase(env.DB_EXTENSIONS); for (const message of batch.messages) { if ( !message.body || @@ -50,19 +50,25 @@ export async function consumePackageJobs( continue; } try { - const result = await processPackageJob(jobs, message.body.jobId, (job) => - prepareQueuedPackage(packagePreparation(env), job) + const result = await processPackagingJob( + jobs, + message.body.jobId, + (job) => prepareQueuedPackage(packagePreparation(env), job) ); if (result === "busy") message.retry({ delaySeconds: 1200 }); else if (result === "retry") message.retry({ delaySeconds: 60 }); else message.ack(); - } catch { + } catch (error) { + logError("extensions/packages", "Package consumer failed", { + jobId: message.body.jobId, + error: error instanceof Error ? error.message : String(error) + }); message.retry({ delaySeconds: 60 }); } } } -export async function recoverPackageJobs(env: CloudflareBindings) { +export async function recoverPackagingJobs(env: CloudflareBindings) { const mode = createCloudflareBindings(env).environment.get( "EXTENSION_PACKAGES_MODE" ); @@ -71,14 +77,14 @@ export async function recoverPackageJobs(env: CloudflareBindings) { !["local", "enabled"].includes(mode ?? "") ) return; - const jobs = new PackageJobsDatabase(env.DB_EXTENSIONS); + const jobs = new PackagingJobsDatabase(env.DB_EXTENSIONS); await jobs.expire(); const queue = createCloudflareBindings(env).queues!.EXTENSION_PACKAGE_QUEUE; for (const id of await jobs.pending()) { try { - await dispatchPackageJob(jobs, queue, id); + await dispatchPackagingJob(jobs, queue, id); } catch (error) { - logError("extensions/packages", "Package job dispatch failed", { + logError("extensions/packages", "Packaging job dispatch failed", { jobId: id, error: error instanceof Error ? error.message : String(error) }); diff --git a/src/lib/adapters/cloudflare/queue.ts b/services/api/src/lib/adapters/cloudflare/queue.ts similarity index 100% rename from src/lib/adapters/cloudflare/queue.ts rename to services/api/src/lib/adapters/cloudflare/queue.ts diff --git a/src/lib/adapters/node/cache.ts b/services/api/src/lib/adapters/node/cache.ts similarity index 100% rename from src/lib/adapters/node/cache.ts rename to services/api/src/lib/adapters/node/cache.ts diff --git a/src/lib/adapters/node/environment.ts b/services/api/src/lib/adapters/node/environment.ts similarity index 100% rename from src/lib/adapters/node/environment.ts rename to services/api/src/lib/adapters/node/environment.ts diff --git a/src/lib/adapters/node/index.ts b/services/api/src/lib/adapters/node/index.ts similarity index 100% rename from src/lib/adapters/node/index.ts rename to services/api/src/lib/adapters/node/index.ts diff --git a/src/lib/auth/bearer-assertion.ts b/services/api/src/lib/auth/bearer-assertion.ts similarity index 99% rename from src/lib/auth/bearer-assertion.ts rename to services/api/src/lib/auth/bearer-assertion.ts index df38571b..377b7856 100644 --- a/src/lib/auth/bearer-assertion.ts +++ b/services/api/src/lib/auth/bearer-assertion.ts @@ -1,5 +1,5 @@ import { verify as verifyJwt } from "hono/jwt"; -import { logWarn } from "../logger"; +import { logWarn } from "@fossbilling/logging"; import { AuthPrincipal, TokenVerifier } from "./interfaces"; const CLOCK_SKEW_SECONDS = 5; diff --git a/src/lib/auth/index.ts b/services/api/src/lib/auth/index.ts similarity index 100% rename from src/lib/auth/index.ts rename to services/api/src/lib/auth/index.ts diff --git a/src/lib/auth/interfaces.ts b/services/api/src/lib/auth/interfaces.ts similarity index 95% rename from src/lib/auth/interfaces.ts rename to services/api/src/lib/auth/interfaces.ts index 9865b87b..863a81da 100644 --- a/src/lib/auth/interfaces.ts +++ b/services/api/src/lib/auth/interfaces.ts @@ -21,6 +21,6 @@ export type AuthPrincipal = export interface TokenVerifier { verify( token: string, - platform: PlatformContext + platform: Pick ): Promise; } diff --git a/src/lib/auth/m2m-bearer.ts b/services/api/src/lib/auth/m2m-bearer.ts similarity index 98% rename from src/lib/auth/m2m-bearer.ts rename to services/api/src/lib/auth/m2m-bearer.ts index a3316b8b..472181ed 100644 --- a/src/lib/auth/m2m-bearer.ts +++ b/services/api/src/lib/auth/m2m-bearer.ts @@ -1,6 +1,6 @@ import { verifyBearerToken } from "better-auth/oauth2"; import type { AuthPrincipal, TokenVerifier } from "./interfaces"; -import { logWarn } from "../logger"; +import { logWarn } from "@fossbilling/logging"; // Machine-to-machine bearer tokens minted by the central auth service's // client_credentials grant (FOSSBilling/auth). Tokens carrying a `resource` diff --git a/src/lib/auth/middleware.ts b/services/api/src/lib/auth/middleware.ts similarity index 100% rename from src/lib/auth/middleware.ts rename to services/api/src/lib/auth/middleware.ts diff --git a/src/lib/cache.ts b/services/api/src/lib/cache.ts similarity index 100% rename from src/lib/cache.ts rename to services/api/src/lib/cache.ts diff --git a/src/lib/context.ts b/services/api/src/lib/context.ts similarity index 51% rename from src/lib/context.ts rename to services/api/src/lib/context.ts index 5210bb67..29f20d3e 100644 --- a/src/lib/context.ts +++ b/services/api/src/lib/context.ts @@ -1,16 +1,17 @@ -import { Context } from "hono"; import { ICache, IMessageQueue, IPlatformBindings } from "./interfaces"; export interface PlatformContext { getQueue(name: string): IMessageQueue; getCache(name: string): ICache; getEnv(key: string): string | undefined; - readonly raw: Context; + fetchService(name: string, request: Request): Promise; + checkRateLimit(name: string, key: string): Promise; + waitUntil(promise: Promise): void; } export function createPlatformContext( - c: Context, - bindings: IPlatformBindings + bindings: IPlatformBindings, + waitUntil: (promise: Promise) => void ): PlatformContext { return { getQueue(name: string): IMessageQueue { @@ -30,8 +31,18 @@ export function createPlatformContext( return bindings.environment.get(key); }, - get raw(): Context { - return c; - } + async fetchService(name: string, request: Request): Promise { + const service = bindings.services?.[name]; + if (!service) throw new Error(`Service binding '${name}' not found`); + return service(request); + }, + + async checkRateLimit(name: string, key: string): Promise { + const limiter = bindings.rateLimiters?.[name]; + if (!limiter) throw new Error(`Rate limiter binding '${name}' not found`); + return limiter(key); + }, + + waitUntil }; } diff --git a/src/lib/db.ts b/services/api/src/lib/db.ts similarity index 100% rename from src/lib/db.ts rename to services/api/src/lib/db.ts diff --git a/src/lib/github-errors.ts b/services/api/src/lib/github-errors.ts similarity index 100% rename from src/lib/github-errors.ts rename to services/api/src/lib/github-errors.ts diff --git a/src/lib/hash.ts b/services/api/src/lib/hash.ts similarity index 100% rename from src/lib/hash.ts rename to services/api/src/lib/hash.ts diff --git a/src/lib/interfaces.ts b/services/api/src/lib/interfaces.ts similarity index 84% rename from src/lib/interfaces.ts rename to services/api/src/lib/interfaces.ts index a4523baa..3ce40e8a 100644 --- a/src/lib/interfaces.ts +++ b/services/api/src/lib/interfaces.ts @@ -27,6 +27,8 @@ export interface IMessageQueue { export interface IPlatformBindings { queues?: Record; + services?: Record Promise>; + rateLimiters?: Record Promise>; caches: Record; environment: IEnvironment; } diff --git a/src/lib/json.ts b/services/api/src/lib/json.ts similarity index 100% rename from src/lib/json.ts rename to services/api/src/lib/json.ts diff --git a/src/lib/middleware.ts b/services/api/src/lib/middleware.ts similarity index 79% rename from src/lib/middleware.ts rename to services/api/src/lib/middleware.ts index 404eb5ca..46b1e304 100644 --- a/src/lib/middleware.ts +++ b/services/api/src/lib/middleware.ts @@ -9,10 +9,11 @@ declare module "hono" { } export function platformMiddleware( - bindings: IPlatformBindings + bindings: IPlatformBindings, + waitUntil: (promise: Promise) => void ): MiddlewareHandler { return async (c, next) => { - c.set("platform", createPlatformContext(c, bindings)); + c.set("platform", createPlatformContext(bindings, waitUntil)); await next(); }; } diff --git a/src/lib/pagination.ts b/services/api/src/lib/pagination.ts similarity index 100% rename from src/lib/pagination.ts rename to services/api/src/lib/pagination.ts diff --git a/src/lib/releases.ts b/services/api/src/lib/releases.ts similarity index 100% rename from src/lib/releases.ts rename to services/api/src/lib/releases.ts diff --git a/src/services/central-alerts/v1/README.md b/services/api/src/services/central-alerts/v1/README.md similarity index 100% rename from src/services/central-alerts/v1/README.md rename to services/api/src/services/central-alerts/v1/README.md diff --git a/src/services/central-alerts/v1/database.ts b/services/api/src/services/central-alerts/v1/database.ts similarity index 100% rename from src/services/central-alerts/v1/database.ts rename to services/api/src/services/central-alerts/v1/database.ts diff --git a/src/services/central-alerts/v1/db/migrations/0000_create_central_alerts.sql b/services/api/src/services/central-alerts/v1/db/migrations/0000_create_central_alerts.sql similarity index 100% rename from src/services/central-alerts/v1/db/migrations/0000_create_central_alerts.sql rename to services/api/src/services/central-alerts/v1/db/migrations/0000_create_central_alerts.sql diff --git a/src/services/central-alerts/v1/db/migrations/0001_seed_initial_alert.sql b/services/api/src/services/central-alerts/v1/db/migrations/0001_seed_initial_alert.sql similarity index 100% rename from src/services/central-alerts/v1/db/migrations/0001_seed_initial_alert.sql rename to services/api/src/services/central-alerts/v1/db/migrations/0001_seed_initial_alert.sql diff --git a/src/services/central-alerts/v1/db/migrations/meta/0000_snapshot.json b/services/api/src/services/central-alerts/v1/db/migrations/meta/0000_snapshot.json similarity index 100% rename from src/services/central-alerts/v1/db/migrations/meta/0000_snapshot.json rename to services/api/src/services/central-alerts/v1/db/migrations/meta/0000_snapshot.json diff --git a/src/services/central-alerts/v1/db/migrations/meta/0001_snapshot.json b/services/api/src/services/central-alerts/v1/db/migrations/meta/0001_snapshot.json similarity index 100% rename from src/services/central-alerts/v1/db/migrations/meta/0001_snapshot.json rename to services/api/src/services/central-alerts/v1/db/migrations/meta/0001_snapshot.json diff --git a/src/services/central-alerts/v1/db/migrations/meta/_journal.json b/services/api/src/services/central-alerts/v1/db/migrations/meta/_journal.json similarity index 100% rename from src/services/central-alerts/v1/db/migrations/meta/_journal.json rename to services/api/src/services/central-alerts/v1/db/migrations/meta/_journal.json diff --git a/src/services/central-alerts/v1/db/schema.ts b/services/api/src/services/central-alerts/v1/db/schema.ts similarity index 100% rename from src/services/central-alerts/v1/db/schema.ts rename to services/api/src/services/central-alerts/v1/db/schema.ts diff --git a/src/services/central-alerts/v1/index.ts b/services/api/src/services/central-alerts/v1/index.ts similarity index 94% rename from src/services/central-alerts/v1/index.ts rename to services/api/src/services/central-alerts/v1/index.ts index 60a16a00..9bba90ac 100644 --- a/src/services/central-alerts/v1/index.ts +++ b/services/api/src/services/central-alerts/v1/index.ts @@ -1,9 +1,10 @@ +import { getPlatform } from "../../../lib/middleware"; import { Hono } from "hono"; import { cors } from "hono/cors"; import { trimTrailingSlash } from "hono/trailing-slash"; import { CentralAlertsDatabase } from "./database"; import { getCentralAlertsDb } from "../../../lib/db"; -import { logError } from "../../../lib/logger"; +import { logError } from "@fossbilling/logging"; import { parseLegacyPagination } from "../../../lib/pagination"; const centralAlertsV1 = new Hono<{ Bindings: CloudflareBindings }>(); @@ -86,7 +87,7 @@ centralAlertsV1.get("/list", async (c) => { error: null }); if (edgeCache) { - c.executionCtx.waitUntil(edgeCache.put(cacheUrl.href, response.clone())); + getPlatform(c).waitUntil(edgeCache.put(cacheUrl.href, response.clone())); } return response; }); diff --git a/src/services/central-alerts/v1/interfaces.ts b/services/api/src/services/central-alerts/v1/interfaces.ts similarity index 100% rename from src/services/central-alerts/v1/interfaces.ts rename to services/api/src/services/central-alerts/v1/interfaces.ts diff --git a/src/services/extensions/v1/README.md b/services/api/src/services/extensions/v1/README.md similarity index 100% rename from src/services/extensions/v1/README.md rename to services/api/src/services/extensions/v1/README.md diff --git a/src/services/extensions/v1/cloudflare-rules.json b/services/api/src/services/extensions/v1/cloudflare-rules.json similarity index 100% rename from src/services/extensions/v1/cloudflare-rules.json rename to services/api/src/services/extensions/v1/cloudflare-rules.json diff --git a/src/services/extensions/v1/database.ts b/services/api/src/services/extensions/v1/database.ts similarity index 100% rename from src/services/extensions/v1/database.ts rename to services/api/src/services/extensions/v1/database.ts diff --git a/src/services/extensions/v1/db/schema.sql b/services/api/src/services/extensions/v1/db/schema.sql similarity index 100% rename from src/services/extensions/v1/db/schema.sql rename to services/api/src/services/extensions/v1/db/schema.sql diff --git a/src/services/extensions/v1/index.ts b/services/api/src/services/extensions/v1/index.ts similarity index 92% rename from src/services/extensions/v1/index.ts rename to services/api/src/services/extensions/v1/index.ts index 74d254aa..8a0f38ea 100644 --- a/src/services/extensions/v1/index.ts +++ b/services/api/src/services/extensions/v1/index.ts @@ -1,3 +1,4 @@ +import { getPlatform } from "../../../lib/middleware"; import { Hono, type Context } from "hono"; import { cors } from "hono/cors"; import { trimTrailingSlash } from "hono/trailing-slash"; @@ -32,6 +33,7 @@ extensionsV1.use("/*", trimTrailingSlash()); // nondeterministic row order. extensionsV1.get("/list", async (c) => { + const platform = getPlatform(c); const db = new ExtensionsDatabase(getExtensionsDb(c.env.DB_EXTENSIONS)); const type = c.req.query("type"); @@ -56,7 +58,7 @@ extensionsV1.get("/list", async (c) => { async (): Promise => { let cached: string | null = null; try { - cached = await c.env.CACHE_KV.get(cacheKey); + cached = await platform.getCache("CACHE_KV").get(cacheKey); } catch { // Fall through to the D1 build. } @@ -77,10 +79,13 @@ extensionsV1.get("/list", async (c) => { } : {}) }); - c.executionCtx.waitUntil( - c.env.CACHE_KV.put(cacheKey, serialized, { - expirationTtl: LIST_CACHE_TTL_SECONDS - }).catch(() => {}) + platform.waitUntil( + getPlatform(c) + .getCache("CACHE_KV") + .put(cacheKey, serialized, { + expirationTtl: LIST_CACHE_TTL_SECONDS + }) + .catch(() => {}) ); return serialized; } diff --git a/src/services/extensions/v1/interfaces.ts b/services/api/src/services/extensions/v1/interfaces.ts similarity index 100% rename from src/services/extensions/v1/interfaces.ts rename to services/api/src/services/extensions/v1/interfaces.ts diff --git a/src/services/extensions/v1/list-cache.ts b/services/api/src/services/extensions/v1/list-cache.ts similarity index 96% rename from src/services/extensions/v1/list-cache.ts rename to services/api/src/services/extensions/v1/list-cache.ts index 6e276f8b..af734243 100644 --- a/src/services/extensions/v1/list-cache.ts +++ b/services/api/src/services/extensions/v1/list-cache.ts @@ -1,3 +1,4 @@ +import type { ICache } from "../../../lib/interfaces"; // Cache-key surface for the v1 list body cache (see index.ts). Kept separate // so extensions/v2's revalidateExtensions can invalidate it after extension directory // mutations without importing the v1 app module (which would drag the whole @@ -27,7 +28,7 @@ export function listCacheKey( export const LIST_CACHE_TTL_SECONDS = 60; export function invalidateListCache( - kv: KVNamespace, + kv: ICache, waitUntil: (promise: Promise) => void ): void { // Mirrors listCacheKey's encoding: the "all" sentinel for the unfiltered diff --git a/src/services/extensions/v2/PACKAGING.md b/services/api/src/services/extensions/v2/PACKAGING.md similarity index 85% rename from src/services/extensions/v2/PACKAGING.md rename to services/api/src/services/extensions/v2/PACKAGING.md index 7cabbd67..f7e51255 100644 --- a/src/services/extensions/v2/PACKAGING.md +++ b/services/api/src/services/extensions/v2/PACKAGING.md @@ -14,11 +14,12 @@ Run the API through the normal `npm run dev` command. Use ignored `.dev.vars` for local credentials, set `ENVIRONMENT=local` and `EXTENSION_PACKAGES_MODE=local`, and set `EXTENSION_PACKAGES_BASE_URL` to the local API's `/extensions/v2` URL. Apply local migrations with `npm run db:migrate:all:local`. The configured Queue -and Images bindings run locally under Wrangler; B2 requests use the configured -buckets and development keys, so use a separate key and package object prefix. +runs locally under Wrangler; start the Node packager as described below. B2 +requests use the configured buckets and development keys, so use a separate key +and package object prefix. -`npm run test:all` covers the package lifecycle through the real API routes and -D1 repository, with mocked S3 responses and emulated image processing. Node tests +From the repository root, `npm test` covers the package lifecycle through the real API routes and +D1 repository, with mocked S3 responses and prepared image fixtures. Node tests also exercise package preparation and B2 storage without a Cloudflare runtime. ## API and review @@ -32,7 +33,7 @@ The OpenAPI reference is available at `/extensions/v2/docs`. `extensions:push` and an active mapped developer binding. Moderators may rebuild published stored source for a correction, but cannot upload new files for another developer. -- `GET /extensions/v2/package-jobs/{jobId}` returns `queued`, `processing`, `ready` +- `GET /extensions/v2/packaging-jobs/{jobId}` returns `queued`, `processing`, `ready` or `failed`. Ready jobs include the artifact; failed jobs include an actionable error. Reads require the current owning developer/publisher or a moderator, and are never publicly cached. Identical requests reuse queued/completed work. @@ -63,12 +64,15 @@ processing. The staged upload becomes the immutable source object after validati so it is not uploaded to package storage a second time. The generic `IMessageQueue` producer interface lives in `src/lib/interfaces.ts`; -`CloudflareQueueAdapter` implements it. `processPackageJob` depends on a -`PackageJobRepository` and a preparation callback, not Cloudflare MessageBatch, -acknowledgements or retry APIs. Package preparation accepts a `PackageRepository`, `PackageStore` and -`PackageImageProcessor`. SQL and guarded writes live in `db/package-builds.ts`; -Cloudflare bindings, queue delivery and image processing live under -`src/lib/adapters/cloudflare/`, and the B2 store lives under `src/lib/adapters/b2/`. +`CloudflareQueueAdapter` implements it. `processPackagingJob` depends on a +`PackagingJobRepository` and a preparation callback, not Cloudflare MessageBatch, +acknowledgements or retry APIs. Package preparation accepts a `PackageRepository`, +`PackageStore` and `PackageProcessor`. SQL and guarded writes live in `db/package-builds.ts`; +Cloudflare bindings and queue delivery live under `src/lib/adapters/cloudflare/`. +The `@fossbilling/extension-packaging` workspace defines the shared contracts and adapters. +A `PackageProcessor` handles ZIP validation, manifest replacement and image preparation. +The API calls the shared HTTP adapter; the Netlify entry point runs the +same Node handler used locally. The B2 store lives under `libraries/extension-packaging/src/adapters/b2.ts`. The package workflow contains no Cloudflare binding types or SDK calls. A different runtime supplies these interfaces and its own queue consumer; the rest of the API still has existing platform integrations outside this package workflow. @@ -102,7 +106,10 @@ directory-only changes such as README edits keep the existing package. Approval publishes the revision and its artifacts in one D1 transaction. Older approved artifacts retain their URLs and checksums. -Object reservations precede uploads. Unique constraints deduplicate concurrent +Object reservations precede uploads. Before external processing, the API reserves at most +17 MiB for the archive and 4 MiB for each possible icon; the archive reservation shrinks +to its actual size after processing. This headroom keeps interrupted runner writes +accounted for within the owner's quota. Unique constraints deduplicate concurrent builds; failed writes remain recorded for cleanup. Cleanup claims unreferenced objects before deletion, and builds cannot reference claimed objects. The per-account allowance is 256 objects / 512 MiB, including abandoned reservations @@ -128,10 +135,11 @@ UUIDv7 draft identity after package validation, before the first icon is stored. The first review submission reuses that draft identity. SVGs are sanitized with Cloudflare svg-hush through the pinned `@zeropress/svg-hush` -Workers adapter and served directly. SVG inputs are bounded by the ZIP expansion -limit; there is no separate SVG size cap. Raster inputs use the local `PACKAGE_IMAGES` binding to produce +Node adapter and served directly. SVG inputs are bounded by the ZIP expansion +limit and the sanitizer's built-in input/output limits. Raster inputs use Sharp in the Node packager to produce AVIF and PNG variants within 256 × 256 pixels, preserving proportions and -transparency without enlarging small images. Requests prefer SVG when present, +transparency without enlarging small images. Raster inputs are limited to 16,777,216 pixels. +Requests prefer SVG when present, otherwise AVIF when explicitly accepted, with PNG fallback. They read stored images directly and never reopen a ZIP. @@ -259,8 +267,8 @@ Before deployment (including an automatic deployment after merge): the private package bucket. Supply separate `EXTENSION_ASSETS_S3_ACCESS_KEY_ID` and `EXTENSION_ASSETS_S3_SECRET_ACCESS_KEY` for the public asset bucket. Development keys in `.dev.vars` are not production secrets. -3. Confirm the Images binding is enabled for the account, the scoped CDN security - headers remain configured, and B2 caps/alerts are in place. +3. Configure the packager URL and shared secret described below, confirm the + scoped CDN security headers remain configured, and keep B2 caps/alerts in place. 4. Apply the package and job tables through the normal D1 deployment workflow. 5. After queue-consumer capacity and moderation requirements are agreed, explicitly set `EXTENSION_PACKAGES_MODE=enabled` in the reviewed production configuration. @@ -270,7 +278,7 @@ Before deployment (including an automatic deployment after merge): Production requires HTTPS download URLs with the `/extensions/v2` base path, complete credentials for both B2 buckets, an explicit non-`local` ZIP object prefix, -a public asset URL and the Images binding. An incomplete configuration fails with 503; +a public asset URL and an authenticated extension packager. An incomplete configuration fails with 503; it cannot fall back to local ZIP storage. Private package objects use the configured prefix. Public image objects use the UUID-based layout described above. @@ -280,5 +288,15 @@ it is not an upload-only pause switch. Prepared CDN images remain accessible ind Before enabling package mode, measure admission and queue-consumer invocations separately: CPU and wall duration, memory failures, queue delay, retries, and -storage/image-service latency. Include near-limit ZIPs and SVG/raster icons. +storage and packager latency. Include near-limit ZIPs and SVG/raster icons. Local functional tests do not establish production capacity or billing. + +## Extension packager + +The API dispatches ZIP and image work to a separately deployed Node service through +an authenticated HTTP adapter. Requests and responses contain metadata and private +B2 object keys, not ZIP bytes. The API retains ownership, leases and publication. + +See the [packager setup and deployment guide](../../../../../extension-packager/README.md) for Netlify configuration, +local development, credentials, deadlines and runtime verification. The shared +contracts and portable ZIP implementation live in `libraries/extension-packaging`. diff --git a/src/services/extensions/v2/README.md b/services/api/src/services/extensions/v2/README.md similarity index 100% rename from src/services/extensions/v2/README.md rename to services/api/src/services/extensions/v2/README.md diff --git a/src/services/extensions/v2/db/badges.ts b/services/api/src/services/extensions/v2/db/badges.ts similarity index 100% rename from src/services/extensions/v2/db/badges.ts rename to services/api/src/services/extensions/v2/db/badges.ts diff --git a/src/services/extensions/v2/db/batch.ts b/services/api/src/services/extensions/v2/db/batch.ts similarity index 100% rename from src/services/extensions/v2/db/batch.ts rename to services/api/src/services/extensions/v2/db/batch.ts diff --git a/src/services/extensions/v2/db/claim-verification-budget.ts b/services/api/src/services/extensions/v2/db/claim-verification-budget.ts similarity index 100% rename from src/services/extensions/v2/db/claim-verification-budget.ts rename to services/api/src/services/extensions/v2/db/claim-verification-budget.ts diff --git a/src/services/extensions/v2/db/columns.ts b/services/api/src/services/extensions/v2/db/columns.ts similarity index 100% rename from src/services/extensions/v2/db/columns.ts rename to services/api/src/services/extensions/v2/db/columns.ts diff --git a/src/services/extensions/v2/db/cursor.ts b/services/api/src/services/extensions/v2/db/cursor.ts similarity index 100% rename from src/services/extensions/v2/db/cursor.ts rename to services/api/src/services/extensions/v2/db/cursor.ts diff --git a/src/services/extensions/v2/db/developer-approval.ts b/services/api/src/services/extensions/v2/db/developer-approval.ts similarity index 100% rename from src/services/extensions/v2/db/developer-approval.ts rename to services/api/src/services/extensions/v2/db/developer-approval.ts diff --git a/src/services/extensions/v2/db/developer-claims.ts b/services/api/src/services/extensions/v2/db/developer-claims.ts similarity index 100% rename from src/services/extensions/v2/db/developer-claims.ts rename to services/api/src/services/extensions/v2/db/developer-claims.ts diff --git a/src/services/extensions/v2/db/developer-profiles.ts b/services/api/src/services/extensions/v2/db/developer-profiles.ts similarity index 100% rename from src/services/extensions/v2/db/developer-profiles.ts rename to services/api/src/services/extensions/v2/db/developer-profiles.ts diff --git a/src/services/extensions/v2/db/developer-transfers.ts b/services/api/src/services/extensions/v2/db/developer-transfers.ts similarity index 100% rename from src/services/extensions/v2/db/developer-transfers.ts rename to services/api/src/services/extensions/v2/db/developer-transfers.ts diff --git a/src/services/extensions/v2/db/errors.ts b/services/api/src/services/extensions/v2/db/errors.ts similarity index 99% rename from src/services/extensions/v2/db/errors.ts rename to services/api/src/services/extensions/v2/db/errors.ts index 7ad32e3a..02a30bfc 100644 --- a/src/services/extensions/v2/db/errors.ts +++ b/services/api/src/services/extensions/v2/db/errors.ts @@ -2,7 +2,7 @@ import { DatabaseError } from "../../../../lib/interfaces"; import { ExtensionsDb } from "../../../../lib/db"; import { UsersDatabase } from "./users"; import { DatabaseResult } from "../../../../lib/interfaces"; -import { logInfo, logError } from "../../../../lib/logger"; +import { logInfo, logError } from "@fossbilling/logging"; // Drizzle wraps driver errors; constraint classifiers inspect the cause chain. export function errorMessageChain(error: unknown): string { diff --git a/src/services/extensions/v2/db/extensions.ts b/services/api/src/services/extensions/v2/db/extensions.ts similarity index 100% rename from src/services/extensions/v2/db/extensions.ts rename to services/api/src/services/extensions/v2/db/extensions.ts diff --git a/src/services/extensions/v2/db/manifest-preflight.sql b/services/api/src/services/extensions/v2/db/manifest-preflight.sql similarity index 100% rename from src/services/extensions/v2/db/manifest-preflight.sql rename to services/api/src/services/extensions/v2/db/manifest-preflight.sql diff --git a/src/services/extensions/v2/db/migrations/0000_bootstrap_users.sql b/services/api/src/services/extensions/v2/db/migrations/0000_bootstrap_users.sql similarity index 100% rename from src/services/extensions/v2/db/migrations/0000_bootstrap_users.sql rename to services/api/src/services/extensions/v2/db/migrations/0000_bootstrap_users.sql diff --git a/src/services/extensions/v2/db/migrations/0001_add_v2_tables.sql b/services/api/src/services/extensions/v2/db/migrations/0001_add_v2_tables.sql similarity index 100% rename from src/services/extensions/v2/db/migrations/0001_add_v2_tables.sql rename to services/api/src/services/extensions/v2/db/migrations/0001_add_v2_tables.sql diff --git a/src/services/extensions/v2/db/migrations/0002_add_author_approval.sql b/services/api/src/services/extensions/v2/db/migrations/0002_add_author_approval.sql similarity index 100% rename from src/services/extensions/v2/db/migrations/0002_add_author_approval.sql rename to services/api/src/services/extensions/v2/db/migrations/0002_add_author_approval.sql diff --git a/src/services/extensions/v2/db/migrations/0003_add_author_profile_fields.sql b/services/api/src/services/extensions/v2/db/migrations/0003_add_author_profile_fields.sql similarity index 100% rename from src/services/extensions/v2/db/migrations/0003_add_author_profile_fields.sql rename to services/api/src/services/extensions/v2/db/migrations/0003_add_author_profile_fields.sql diff --git a/src/services/extensions/v2/db/migrations/0004_add_authors_owner_unique.sql b/services/api/src/services/extensions/v2/db/migrations/0004_add_authors_owner_unique.sql similarity index 100% rename from src/services/extensions/v2/db/migrations/0004_add_authors_owner_unique.sql rename to services/api/src/services/extensions/v2/db/migrations/0004_add_authors_owner_unique.sql diff --git a/src/services/extensions/v2/db/migrations/0005_add_author_history.sql b/services/api/src/services/extensions/v2/db/migrations/0005_add_author_history.sql similarity index 100% rename from src/services/extensions/v2/db/migrations/0005_add_author_history.sql rename to services/api/src/services/extensions/v2/db/migrations/0005_add_author_history.sql diff --git a/src/services/extensions/v2/db/migrations/0006_add_author_transfers.sql b/services/api/src/services/extensions/v2/db/migrations/0006_add_author_transfers.sql similarity index 100% rename from src/services/extensions/v2/db/migrations/0006_add_author_transfers.sql rename to services/api/src/services/extensions/v2/db/migrations/0006_add_author_transfers.sql diff --git a/src/services/extensions/v2/db/migrations/0007_add_author_claims.sql b/services/api/src/services/extensions/v2/db/migrations/0007_add_author_claims.sql similarity index 100% rename from src/services/extensions/v2/db/migrations/0007_add_author_claims.sql rename to services/api/src/services/extensions/v2/db/migrations/0007_add_author_claims.sql diff --git a/src/services/extensions/v2/db/migrations/0008_rename_authors_to_developers.sql b/services/api/src/services/extensions/v2/db/migrations/0008_rename_authors_to_developers.sql similarity index 100% rename from src/services/extensions/v2/db/migrations/0008_rename_authors_to_developers.sql rename to services/api/src/services/extensions/v2/db/migrations/0008_rename_authors_to_developers.sql diff --git a/src/services/extensions/v2/db/migrations/0009_drop_developer_history_fk.sql b/services/api/src/services/extensions/v2/db/migrations/0009_drop_developer_history_fk.sql similarity index 100% rename from src/services/extensions/v2/db/migrations/0009_drop_developer_history_fk.sql rename to services/api/src/services/extensions/v2/db/migrations/0009_drop_developer_history_fk.sql diff --git a/src/services/extensions/v2/db/migrations/0010_drop_developer_bio.sql b/services/api/src/services/extensions/v2/db/migrations/0010_drop_developer_bio.sql similarity index 100% rename from src/services/extensions/v2/db/migrations/0010_drop_developer_bio.sql rename to services/api/src/services/extensions/v2/db/migrations/0010_drop_developer_bio.sql diff --git a/src/services/extensions/v2/db/migrations/0011_harden_review_workflows.sql b/services/api/src/services/extensions/v2/db/migrations/0011_harden_review_workflows.sql similarity index 100% rename from src/services/extensions/v2/db/migrations/0011_harden_review_workflows.sql rename to services/api/src/services/extensions/v2/db/migrations/0011_harden_review_workflows.sql diff --git a/src/services/extensions/v2/db/migrations/0012_add_developer_claims_github_verification.sql b/services/api/src/services/extensions/v2/db/migrations/0012_add_developer_claims_github_verification.sql similarity index 100% rename from src/services/extensions/v2/db/migrations/0012_add_developer_claims_github_verification.sql rename to services/api/src/services/extensions/v2/db/migrations/0012_add_developer_claims_github_verification.sql diff --git a/src/services/extensions/v2/db/migrations/0013_add_developers_github_verification.sql b/services/api/src/services/extensions/v2/db/migrations/0013_add_developers_github_verification.sql similarity index 100% rename from src/services/extensions/v2/db/migrations/0013_add_developers_github_verification.sql rename to services/api/src/services/extensions/v2/db/migrations/0013_add_developers_github_verification.sql diff --git a/src/services/extensions/v2/db/migrations/0014_drizzle_baseline.sql b/services/api/src/services/extensions/v2/db/migrations/0014_drizzle_baseline.sql similarity index 100% rename from src/services/extensions/v2/db/migrations/0014_drizzle_baseline.sql rename to services/api/src/services/extensions/v2/db/migrations/0014_drizzle_baseline.sql diff --git a/src/services/extensions/v2/db/migrations/0015_add_developer_github_verified_at.sql b/services/api/src/services/extensions/v2/db/migrations/0015_add_developer_github_verified_at.sql similarity index 100% rename from src/services/extensions/v2/db/migrations/0015_add_developer_github_verified_at.sql rename to services/api/src/services/extensions/v2/db/migrations/0015_add_developer_github_verified_at.sql diff --git a/src/services/extensions/v2/db/migrations/0016_add_developer_github_url_verified.sql b/services/api/src/services/extensions/v2/db/migrations/0016_add_developer_github_url_verified.sql similarity index 100% rename from src/services/extensions/v2/db/migrations/0016_add_developer_github_url_verified.sql rename to services/api/src/services/extensions/v2/db/migrations/0016_add_developer_github_url_verified.sql diff --git a/src/services/extensions/v2/db/migrations/0017_add_developer_url_check_cooldown.sql b/services/api/src/services/extensions/v2/db/migrations/0017_add_developer_url_check_cooldown.sql similarity index 100% rename from src/services/extensions/v2/db/migrations/0017_add_developer_url_check_cooldown.sql rename to services/api/src/services/extensions/v2/db/migrations/0017_add_developer_url_check_cooldown.sql diff --git a/src/services/extensions/v2/db/migrations/0018_add_extension_catalogue_indexes.sql b/services/api/src/services/extensions/v2/db/migrations/0018_add_extension_catalogue_indexes.sql similarity index 100% rename from src/services/extensions/v2/db/migrations/0018_add_extension_catalogue_indexes.sql rename to services/api/src/services/extensions/v2/db/migrations/0018_add_extension_catalogue_indexes.sql diff --git a/src/services/extensions/v2/db/migrations/0019_add_user_deleted_at.sql b/services/api/src/services/extensions/v2/db/migrations/0019_add_user_deleted_at.sql similarity index 100% rename from src/services/extensions/v2/db/migrations/0019_add_user_deleted_at.sql rename to services/api/src/services/extensions/v2/db/migrations/0019_add_user_deleted_at.sql diff --git a/src/services/extensions/v2/db/migrations/0020_check_reserved_route_ids.sql b/services/api/src/services/extensions/v2/db/migrations/0020_check_reserved_route_ids.sql similarity index 100% rename from src/services/extensions/v2/db/migrations/0020_check_reserved_route_ids.sql rename to services/api/src/services/extensions/v2/db/migrations/0020_check_reserved_route_ids.sql diff --git a/src/services/extensions/v2/db/migrations/0021_restructure_extensions_revisions.sql b/services/api/src/services/extensions/v2/db/migrations/0021_restructure_extensions_revisions.sql similarity index 100% rename from src/services/extensions/v2/db/migrations/0021_restructure_extensions_revisions.sql rename to services/api/src/services/extensions/v2/db/migrations/0021_restructure_extensions_revisions.sql diff --git a/src/services/extensions/v2/db/migrations/0022_add_extensions_delisted_at.sql b/services/api/src/services/extensions/v2/db/migrations/0022_add_extensions_delisted_at.sql similarity index 100% rename from src/services/extensions/v2/db/migrations/0022_add_extensions_delisted_at.sql rename to services/api/src/services/extensions/v2/db/migrations/0022_add_extensions_delisted_at.sql diff --git a/src/services/extensions/v2/db/migrations/0023_fast_makkari.sql b/services/api/src/services/extensions/v2/db/migrations/0023_fast_makkari.sql similarity index 100% rename from src/services/extensions/v2/db/migrations/0023_fast_makkari.sql rename to services/api/src/services/extensions/v2/db/migrations/0023_fast_makkari.sql diff --git a/src/services/extensions/v2/db/migrations/0024_claim_verification_budgets.sql b/services/api/src/services/extensions/v2/db/migrations/0024_claim_verification_budgets.sql similarity index 100% rename from src/services/extensions/v2/db/migrations/0024_claim_verification_budgets.sql rename to services/api/src/services/extensions/v2/db/migrations/0024_claim_verification_budgets.sql diff --git a/src/services/extensions/v2/db/migrations/0025_luxuriant_franklin_richards.sql b/services/api/src/services/extensions/v2/db/migrations/0025_luxuriant_franklin_richards.sql similarity index 100% rename from src/services/extensions/v2/db/migrations/0025_luxuriant_franklin_richards.sql rename to services/api/src/services/extensions/v2/db/migrations/0025_luxuriant_franklin_richards.sql diff --git a/src/services/extensions/v2/db/migrations/0026_resource_bounds.sql b/services/api/src/services/extensions/v2/db/migrations/0026_resource_bounds.sql similarity index 100% rename from src/services/extensions/v2/db/migrations/0026_resource_bounds.sql rename to services/api/src/services/extensions/v2/db/migrations/0026_resource_bounds.sql diff --git a/src/services/extensions/v2/db/migrations/0027_bind_developer_profile_generation.sql b/services/api/src/services/extensions/v2/db/migrations/0027_bind_developer_profile_generation.sql similarity index 100% rename from src/services/extensions/v2/db/migrations/0027_bind_developer_profile_generation.sql rename to services/api/src/services/extensions/v2/db/migrations/0027_bind_developer_profile_generation.sql diff --git a/src/services/extensions/v2/db/migrations/0028_m2m_publisher_clients.sql b/services/api/src/services/extensions/v2/db/migrations/0028_m2m_publisher_clients.sql similarity index 100% rename from src/services/extensions/v2/db/migrations/0028_m2m_publisher_clients.sql rename to services/api/src/services/extensions/v2/db/migrations/0028_m2m_publisher_clients.sql diff --git a/src/services/extensions/v2/db/migrations/0029_align_manifest.sql b/services/api/src/services/extensions/v2/db/migrations/0029_align_manifest.sql similarity index 100% rename from src/services/extensions/v2/db/migrations/0029_align_manifest.sql rename to services/api/src/services/extensions/v2/db/migrations/0029_align_manifest.sql diff --git a/src/services/extensions/v2/db/migrations/0030_directory_identity.sql b/services/api/src/services/extensions/v2/db/migrations/0030_directory_identity.sql similarity index 100% rename from src/services/extensions/v2/db/migrations/0030_directory_identity.sql rename to services/api/src/services/extensions/v2/db/migrations/0030_directory_identity.sql diff --git a/src/services/extensions/v2/db/migrations/0031_directory_packages.sql b/services/api/src/services/extensions/v2/db/migrations/0031_directory_packages.sql similarity index 100% rename from src/services/extensions/v2/db/migrations/0031_directory_packages.sql rename to services/api/src/services/extensions/v2/db/migrations/0031_directory_packages.sql diff --git a/src/services/extensions/v2/db/migrations/0032_package_jobs.sql b/services/api/src/services/extensions/v2/db/migrations/0032_package_jobs.sql similarity index 100% rename from src/services/extensions/v2/db/migrations/0032_package_jobs.sql rename to services/api/src/services/extensions/v2/db/migrations/0032_package_jobs.sql diff --git a/src/services/extensions/v2/db/migrations/meta/0014_snapshot.json b/services/api/src/services/extensions/v2/db/migrations/meta/0014_snapshot.json similarity index 100% rename from src/services/extensions/v2/db/migrations/meta/0014_snapshot.json rename to services/api/src/services/extensions/v2/db/migrations/meta/0014_snapshot.json diff --git a/src/services/extensions/v2/db/migrations/meta/0015_snapshot.json b/services/api/src/services/extensions/v2/db/migrations/meta/0015_snapshot.json similarity index 100% rename from src/services/extensions/v2/db/migrations/meta/0015_snapshot.json rename to services/api/src/services/extensions/v2/db/migrations/meta/0015_snapshot.json diff --git a/src/services/extensions/v2/db/migrations/meta/0016_snapshot.json b/services/api/src/services/extensions/v2/db/migrations/meta/0016_snapshot.json similarity index 100% rename from src/services/extensions/v2/db/migrations/meta/0016_snapshot.json rename to services/api/src/services/extensions/v2/db/migrations/meta/0016_snapshot.json diff --git a/src/services/extensions/v2/db/migrations/meta/0017_snapshot.json b/services/api/src/services/extensions/v2/db/migrations/meta/0017_snapshot.json similarity index 100% rename from src/services/extensions/v2/db/migrations/meta/0017_snapshot.json rename to services/api/src/services/extensions/v2/db/migrations/meta/0017_snapshot.json diff --git a/src/services/extensions/v2/db/migrations/meta/0018_snapshot.json b/services/api/src/services/extensions/v2/db/migrations/meta/0018_snapshot.json similarity index 100% rename from src/services/extensions/v2/db/migrations/meta/0018_snapshot.json rename to services/api/src/services/extensions/v2/db/migrations/meta/0018_snapshot.json diff --git a/src/services/extensions/v2/db/migrations/meta/0019_snapshot.json b/services/api/src/services/extensions/v2/db/migrations/meta/0019_snapshot.json similarity index 100% rename from src/services/extensions/v2/db/migrations/meta/0019_snapshot.json rename to services/api/src/services/extensions/v2/db/migrations/meta/0019_snapshot.json diff --git a/src/services/extensions/v2/db/migrations/meta/0021_snapshot.json b/services/api/src/services/extensions/v2/db/migrations/meta/0021_snapshot.json similarity index 100% rename from src/services/extensions/v2/db/migrations/meta/0021_snapshot.json rename to services/api/src/services/extensions/v2/db/migrations/meta/0021_snapshot.json diff --git a/src/services/extensions/v2/db/migrations/meta/0022_snapshot.json b/services/api/src/services/extensions/v2/db/migrations/meta/0022_snapshot.json similarity index 100% rename from src/services/extensions/v2/db/migrations/meta/0022_snapshot.json rename to services/api/src/services/extensions/v2/db/migrations/meta/0022_snapshot.json diff --git a/src/services/extensions/v2/db/migrations/meta/0023_snapshot.json b/services/api/src/services/extensions/v2/db/migrations/meta/0023_snapshot.json similarity index 100% rename from src/services/extensions/v2/db/migrations/meta/0023_snapshot.json rename to services/api/src/services/extensions/v2/db/migrations/meta/0023_snapshot.json diff --git a/src/services/extensions/v2/db/migrations/meta/0024_snapshot.json b/services/api/src/services/extensions/v2/db/migrations/meta/0024_snapshot.json similarity index 100% rename from src/services/extensions/v2/db/migrations/meta/0024_snapshot.json rename to services/api/src/services/extensions/v2/db/migrations/meta/0024_snapshot.json diff --git a/src/services/extensions/v2/db/migrations/meta/0025_snapshot.json b/services/api/src/services/extensions/v2/db/migrations/meta/0025_snapshot.json similarity index 100% rename from src/services/extensions/v2/db/migrations/meta/0025_snapshot.json rename to services/api/src/services/extensions/v2/db/migrations/meta/0025_snapshot.json diff --git a/src/services/extensions/v2/db/migrations/meta/0026_snapshot.json b/services/api/src/services/extensions/v2/db/migrations/meta/0026_snapshot.json similarity index 100% rename from src/services/extensions/v2/db/migrations/meta/0026_snapshot.json rename to services/api/src/services/extensions/v2/db/migrations/meta/0026_snapshot.json diff --git a/src/services/extensions/v2/db/migrations/meta/0027_snapshot.json b/services/api/src/services/extensions/v2/db/migrations/meta/0027_snapshot.json similarity index 100% rename from src/services/extensions/v2/db/migrations/meta/0027_snapshot.json rename to services/api/src/services/extensions/v2/db/migrations/meta/0027_snapshot.json diff --git a/src/services/extensions/v2/db/migrations/meta/0028_snapshot.json b/services/api/src/services/extensions/v2/db/migrations/meta/0028_snapshot.json similarity index 100% rename from src/services/extensions/v2/db/migrations/meta/0028_snapshot.json rename to services/api/src/services/extensions/v2/db/migrations/meta/0028_snapshot.json diff --git a/src/services/extensions/v2/db/migrations/meta/0029_snapshot.json b/services/api/src/services/extensions/v2/db/migrations/meta/0029_snapshot.json similarity index 100% rename from src/services/extensions/v2/db/migrations/meta/0029_snapshot.json rename to services/api/src/services/extensions/v2/db/migrations/meta/0029_snapshot.json diff --git a/src/services/extensions/v2/db/migrations/meta/0030_snapshot.json b/services/api/src/services/extensions/v2/db/migrations/meta/0030_snapshot.json similarity index 100% rename from src/services/extensions/v2/db/migrations/meta/0030_snapshot.json rename to services/api/src/services/extensions/v2/db/migrations/meta/0030_snapshot.json diff --git a/src/services/extensions/v2/db/migrations/meta/0031_snapshot.json b/services/api/src/services/extensions/v2/db/migrations/meta/0031_snapshot.json similarity index 100% rename from src/services/extensions/v2/db/migrations/meta/0031_snapshot.json rename to services/api/src/services/extensions/v2/db/migrations/meta/0031_snapshot.json diff --git a/src/services/extensions/v2/db/migrations/meta/0032_snapshot.json b/services/api/src/services/extensions/v2/db/migrations/meta/0032_snapshot.json similarity index 100% rename from src/services/extensions/v2/db/migrations/meta/0032_snapshot.json rename to services/api/src/services/extensions/v2/db/migrations/meta/0032_snapshot.json diff --git a/src/services/extensions/v2/db/migrations/meta/_journal.json b/services/api/src/services/extensions/v2/db/migrations/meta/_journal.json similarity index 100% rename from src/services/extensions/v2/db/migrations/meta/_journal.json rename to services/api/src/services/extensions/v2/db/migrations/meta/_journal.json diff --git a/src/services/extensions/v2/db/package-builds.ts b/services/api/src/services/extensions/v2/db/package-builds.ts similarity index 96% rename from src/services/extensions/v2/db/package-builds.ts rename to services/api/src/services/extensions/v2/db/package-builds.ts index 6881017b..97b58ce8 100644 --- a/src/services/extensions/v2/db/package-builds.ts +++ b/services/api/src/services/extensions/v2/db/package-builds.ts @@ -10,7 +10,7 @@ import type { ArtifactRow, IconObject } from "../packages/repository"; -import { PackageError } from "../packages/errors"; +import { PackageError } from "@fossbilling/extension-packaging/errors"; export const MAX_PACKAGE_STORAGE_BYTES = 512 * 1024 * 1024; export const MAX_PACKAGE_OBJECTS = 256; @@ -112,6 +112,20 @@ export class PackageBuildDatabase implements PackageRepository { 409 ); } + async setObjectSize(id: string, bytes: number) { + const result = await this.db + .prepare( + "UPDATE package_objects SET bytes=? WHERE id=? AND deleting=0 AND bytes>=?" + ) + .bind(bytes, id, bytes) + .run(); + if (!result.meta.changes) + throw new PackageError( + "PACKAGE_UNAVAILABLE", + "Prepared package exceeds its storage reservation", + 503 + ); + } async removeObject(id: string) { await this.db .prepare("DELETE FROM package_objects WHERE id=?") diff --git a/src/services/extensions/v2/db/packages.ts b/services/api/src/services/extensions/v2/db/packages.ts similarity index 100% rename from src/services/extensions/v2/db/packages.ts rename to services/api/src/services/extensions/v2/db/packages.ts diff --git a/src/services/extensions/v2/db/package-jobs.ts b/services/api/src/services/extensions/v2/db/packaging-jobs.ts similarity index 91% rename from src/services/extensions/v2/db/package-jobs.ts rename to services/api/src/services/extensions/v2/db/packaging-jobs.ts index 0a943ac9..08978092 100644 --- a/src/services/extensions/v2/db/package-jobs.ts +++ b/services/api/src/services/extensions/v2/db/packaging-jobs.ts @@ -1,5 +1,5 @@ -import type { PackageJob, PackageJobRepository } from "../packages/jobs"; -import { PackageError } from "../packages/errors"; +import type { PackagingJob, PackagingJobRepository } from "../packages/jobs"; +import { PackageError } from "@fossbilling/extension-packaging/errors"; interface JobRow { id: string; @@ -9,7 +9,7 @@ interface JobRow { source_id: string | null; upload_object_id: string | null; upload_sha256: string | null; - status: PackageJob["status"]; + status: PackagingJob["status"]; attempts: number; artifact_id: string | null; error_code: string | null; @@ -18,7 +18,7 @@ interface JobRow { created_at: string; updated_at: string; } -const job = (row: JobRow): PackageJob => ({ +const job = (row: JobRow): PackagingJob => ({ id: row.id, fingerprint: row.fingerprint, owner: JSON.parse(row.owner), @@ -40,7 +40,7 @@ const job = (row: JobRow): PackageJob => ({ updatedAt: row.updated_at }); -export class PackageJobsDatabase implements PackageJobRepository { +export class PackagingJobsDatabase implements PackagingJobRepository { constructor(private readonly db: D1Database) {} async get(id: string) { @@ -61,7 +61,7 @@ export class PackageJobsDatabase implements PackageJobRepository { } async create( input: Pick< - PackageJob, + PackagingJob, | "fingerprint" | "owner" | "manifest" @@ -93,7 +93,7 @@ export class PackageJobsDatabase implements PackageJobRepository { if (!result) throw new PackageError( "PACKAGE_ADMISSION", - "Package job allowance is exhausted; try again later", + "Packaging job allowance is exhausted; try again later", 429 ); return result; @@ -119,7 +119,7 @@ export class PackageJobsDatabase implements PackageJobRepository { async fail( id: string, lease: string, - error: NonNullable, + error: NonNullable, retry: boolean ) { await this.db diff --git a/src/services/extensions/v2/db/publisher-clients.ts b/services/api/src/services/extensions/v2/db/publisher-clients.ts similarity index 100% rename from src/services/extensions/v2/db/publisher-clients.ts rename to services/api/src/services/extensions/v2/db/publisher-clients.ts diff --git a/src/services/extensions/v2/db/resource-inventory.sql b/services/api/src/services/extensions/v2/db/resource-inventory.sql similarity index 100% rename from src/services/extensions/v2/db/resource-inventory.sql rename to services/api/src/services/extensions/v2/db/resource-inventory.sql diff --git a/src/services/extensions/v2/db/resource-maintenance.ts b/services/api/src/services/extensions/v2/db/resource-maintenance.ts similarity index 98% rename from src/services/extensions/v2/db/resource-maintenance.ts rename to services/api/src/services/extensions/v2/db/resource-maintenance.ts index a85e0116..c1116e09 100644 --- a/src/services/extensions/v2/db/resource-maintenance.ts +++ b/services/api/src/services/extensions/v2/db/resource-maintenance.ts @@ -1,6 +1,6 @@ import { sql } from "drizzle-orm"; import { ExtensionsDb } from "../../../../lib/db"; -import { logInfo, logWarn } from "../../../../lib/logger"; +import { logInfo, logWarn } from "@fossbilling/logging"; import { sha256Hex } from "../../../../lib/hash"; import { MAX_CONTENT_BYTES, diff --git a/src/services/extensions/v2/db/revisions.ts b/services/api/src/services/extensions/v2/db/revisions.ts similarity index 100% rename from src/services/extensions/v2/db/revisions.ts rename to services/api/src/services/extensions/v2/db/revisions.ts diff --git a/src/services/extensions/v2/db/schema.ts b/services/api/src/services/extensions/v2/db/schema.ts similarity index 99% rename from src/services/extensions/v2/db/schema.ts rename to services/api/src/services/extensions/v2/db/schema.ts index 53630359..e5e0c3d5 100644 --- a/src/services/extensions/v2/db/schema.ts +++ b/services/api/src/services/extensions/v2/db/schema.ts @@ -592,7 +592,7 @@ export const packageIcons = sqliteTable("package_icons", { svgSha256: text("svg_sha256") }); -export const packageJobs = sqliteTable( +export const packagingJobs = sqliteTable( "package_jobs", { id: text("id").primaryKey(), diff --git a/src/services/extensions/v2/db/users.ts b/services/api/src/services/extensions/v2/db/users.ts similarity index 100% rename from src/services/extensions/v2/db/users.ts rename to services/api/src/services/extensions/v2/db/users.ts diff --git a/src/services/extensions/v2/email/config.ts b/services/api/src/services/extensions/v2/email/config.ts similarity index 100% rename from src/services/extensions/v2/email/config.ts rename to services/api/src/services/extensions/v2/email/config.ts diff --git a/src/services/extensions/v2/email/factory.ts b/services/api/src/services/extensions/v2/email/factory.ts similarity index 100% rename from src/services/extensions/v2/email/factory.ts rename to services/api/src/services/extensions/v2/email/factory.ts diff --git a/src/services/extensions/v2/email/mxroute.ts b/services/api/src/services/extensions/v2/email/mxroute.ts similarity index 100% rename from src/services/extensions/v2/email/mxroute.ts rename to services/api/src/services/extensions/v2/email/mxroute.ts diff --git a/src/services/extensions/v2/email/notify.ts b/services/api/src/services/extensions/v2/email/notify.ts similarity index 99% rename from src/services/extensions/v2/email/notify.ts rename to services/api/src/services/extensions/v2/email/notify.ts index 3ee434c1..169ab4a2 100644 --- a/src/services/extensions/v2/email/notify.ts +++ b/services/api/src/services/extensions/v2/email/notify.ts @@ -1,5 +1,5 @@ import type { ExtensionsDb } from "../../../../lib/db"; -import { logError } from "../../../../lib/logger"; +import { logError } from "@fossbilling/logging"; import { createEmailSender, DisabledSender } from "./factory"; import { resolveClaimantEmail, diff --git a/src/services/extensions/v2/email/recipients.ts b/services/api/src/services/extensions/v2/email/recipients.ts similarity index 100% rename from src/services/extensions/v2/email/recipients.ts rename to services/api/src/services/extensions/v2/email/recipients.ts diff --git a/src/services/extensions/v2/email/resend.ts b/services/api/src/services/extensions/v2/email/resend.ts similarity index 100% rename from src/services/extensions/v2/email/resend.ts rename to services/api/src/services/extensions/v2/email/resend.ts diff --git a/src/services/extensions/v2/email/templates.ts b/services/api/src/services/extensions/v2/email/templates.ts similarity index 100% rename from src/services/extensions/v2/email/templates.ts rename to services/api/src/services/extensions/v2/email/templates.ts diff --git a/src/services/extensions/v2/email/types.ts b/services/api/src/services/extensions/v2/email/types.ts similarity index 100% rename from src/services/extensions/v2/email/types.ts rename to services/api/src/services/extensions/v2/email/types.ts diff --git a/src/services/extensions/v2/github/identity.ts b/services/api/src/services/extensions/v2/github/identity.ts similarity index 100% rename from src/services/extensions/v2/github/identity.ts rename to services/api/src/services/extensions/v2/github/identity.ts diff --git a/src/services/extensions/v2/github/verification.ts b/services/api/src/services/extensions/v2/github/verification.ts similarity index 99% rename from src/services/extensions/v2/github/verification.ts rename to services/api/src/services/extensions/v2/github/verification.ts index d33b6da5..5f49fd29 100644 --- a/src/services/extensions/v2/github/verification.ts +++ b/services/api/src/services/extensions/v2/github/verification.ts @@ -5,7 +5,7 @@ import { RateLimitError, NotFoundError } from "../../../../lib/github-errors"; -import { logWarn } from "../../../../lib/logger"; +import { logWarn } from "@fossbilling/logging"; import { Developer } from "../schemas/developers"; import { GithubIdentity } from "../db/users"; diff --git a/src/services/extensions/v2/index.ts b/services/api/src/services/extensions/v2/index.ts similarity index 100% rename from src/services/extensions/v2/index.ts rename to services/api/src/services/extensions/v2/index.ts diff --git a/src/services/extensions/v2/middleware.ts b/services/api/src/services/extensions/v2/middleware.ts similarity index 97% rename from src/services/extensions/v2/middleware.ts rename to services/api/src/services/extensions/v2/middleware.ts index b7d7e6b3..c05d662b 100644 --- a/src/services/extensions/v2/middleware.ts +++ b/services/api/src/services/extensions/v2/middleware.ts @@ -1,12 +1,13 @@ -import { logInfo, logError } from "../../../lib/logger"; +import { getPlatform } from "../../../lib/middleware"; +import { logInfo, logError } from "@fossbilling/logging"; import { MAX_RAW_BODY_BYTES } from "./resource-limits"; import { type Context, type MiddlewareHandler } from "hono"; import { bearerAssertionVerifier, identitySyncAssertionVerifier } from "../../../lib/auth/bearer-assertion"; -import { getAuth, requireAuth } from "../../../lib/auth"; -import type { AuthPrincipal } from "../../../lib/auth"; +import { getAuth, requireAuth } from "../../../lib/auth/index"; +import type { AuthPrincipal } from "../../../lib/auth/index"; import { getExtensionsDb } from "../../../lib/db"; import { sha256Hex } from "../../../lib/hash"; import { UsersDatabase } from "./db/users"; @@ -355,9 +356,10 @@ function paceContentAttempts(identity: "ip" | "account"): MiddlewareHandler { const bucket = identity === "ip" ? "ip" : auth?.scope === "m2m" ? "client" : "account"; try { - const { success } = await c.env.EXTENSION_WRITE_RATE_LIMITER.limit({ - key: `${bucket}:${subject}` - }); + const success = await getPlatform(c).checkRateLimit( + "EXTENSION_WRITE_RATE_LIMITER", + `${bucket}:${subject}` + ); if (!success) { c.header("Retry-After", "60"); logInfo("extensions-v2", "Resource admission rejected", { diff --git a/src/services/extensions/v2/packages/build.ts b/services/api/src/services/extensions/v2/packages/build.ts similarity index 66% rename from src/services/extensions/v2/packages/build.ts rename to services/api/src/services/extensions/v2/packages/build.ts index 371aef56..7290301a 100644 --- a/src/services/extensions/v2/packages/build.ts +++ b/services/api/src/services/extensions/v2/packages/build.ts @@ -1,14 +1,18 @@ +import { sha256 } from "@fossbilling/extension-packaging/checksum"; import { - buildExtensionPackage, - PACKAGE_LIMITS, - sha256 -} from "../package-builder"; + MAX_PACKAGE_OBJECT_BYTES, + MAX_PACKAGE_ICON_BYTES, + type PackageProcessor +} from "@fossbilling/extension-packaging/processor"; import type { Manifest } from "../schemas/manifest"; import { canonicalJson } from "./manifest"; import type { PackageRepository, PackageOwner, IconObject } from "./repository"; -import type { PackageImageProcessor, IconVariant } from "./images"; -import { PackageError } from "./errors"; -import { boundedBytes, type PackageStore } from "./store"; +import type { IconVariant } from "@fossbilling/extension-packaging/images"; +import { PackageError } from "@fossbilling/extension-packaging/errors"; +import { + boundedBytes, + type PackageStore +} from "@fossbilling/extension-packaging/store"; export async function reserveObject( repository: PackageRepository, @@ -41,10 +45,10 @@ export async function buildStoredPackage(input: { store: PackageStore; owner: PackageOwner; manifest: Manifest; - upload?: { bytes: Uint8Array; objectId: string }; + upload?: { sha256: string; objectId: string }; sourceId?: string; baseUrl: string; - images: PackageImageProcessor; + processor: PackageProcessor; }) { const { repository, store, owner, manifest } = input; let source = await repository.findSource( @@ -58,8 +62,7 @@ export async function buildStoredPackage(input: { "Source not found for this extension version", 404 ); - let sourceBytes = input.upload?.bytes; - if (sourceBytes && source && (await sha256(sourceBytes)) !== source.sha256) + if (input.upload && source && input.upload.sha256 !== source.sha256) throw new PackageError( "SOURCE_CHANGED", "This version already has a different source ZIP; use a new extension version", @@ -73,42 +76,56 @@ export async function buildStoredPackage(input: { const existing = await repository.findArtifact(source.id, manifestHash); if (existing) return existing; } - if (!sourceBytes) { - if (!source) - throw new PackageError("NOT_FOUND", "Source ZIP not found", 404); - sourceBytes = await boundedBytes( - await store.get(source.object_id), - PACKAGE_LIMITS.sourceBytes + const upload = + input.upload ?? + (source ? { objectId: source.object_id, sha256: source.sha256 } : null); + if (!upload) throw new PackageError("NOT_FOUND", "Source ZIP not found", 404); + const prepared = source && (await repository.hasIcon(source.id)); + const objectId = crypto.randomUUID(); + const iconIds: [string, string] | undefined = prepared + ? undefined + : [crypto.randomUUID(), crypto.randomUUID()]; + // Reserve worst-case output space before an external runner can write to B2. + // Unused/failed reservations remain discoverable by the normal cleanup job. + await repository.reserveObject( + owner, + manifest.id, + objectId, + MAX_PACKAGE_OBJECT_BYTES + ); + for (const id of iconIds ?? []) + await repository.reserveObject( + owner, + manifest.id, + id, + MAX_PACKAGE_ICON_BYTES ); - if ((await sha256(sourceBytes)) !== source.sha256) + const built = await input.processor.process({ + manifest, + source: { id: upload.objectId, sha256: upload.sha256 }, + archive: objectId, + icons: iconIds + }); + await repository.setObjectSize(objectId, built.archive.bytes); + const variants: IconVariant[] = []; + for (const icon of built.icons) { + const bytes = await boundedBytes( + await store.get(icon.id, icon.sha256), + MAX_PACKAGE_ICON_BYTES + ); + if (bytes.length !== icon.bytes || (await sha256(bytes)) !== icon.sha256) throw new PackageError( - "SOURCE_UNAVAILABLE", - "Stored source failed checksum verification", + "PACKAGE_UNAVAILABLE", + "Prepared icon failed checksum verification", 503 ); + variants.push({ format: icon.format, bytes }); } - let built; - try { - built = await buildExtensionPackage(sourceBytes, manifest); - } catch { - throw new PackageError( - "INVALID_PACKAGE", - "Source ZIP failed package validation", - 422 - ); - } - const prepared = source && (await repository.hasIcon(source.id)); - const variants = - prepared || !built.icon - ? [] - : await input.images.prepare(built.icon, built.iconName === "icon.svg"); if (!source) { - // Without an upload, the stored-source lookup above must have succeeded. - const objectId = input.upload!.objectId; source = await repository.linkSource(owner, manifest, { id: crypto.randomUUID(), - sha256: built.sourceSha256, - object_id: objectId + sha256: upload.sha256, + object_id: upload.objectId }); if (!source) throw new PackageError( @@ -116,7 +133,7 @@ export async function buildStoredPackage(input: { "Ownership changed while building", 403 ); - if (source.sha256 !== built.sourceSha256) + if (source.sha256 !== upload.sha256) throw new PackageError( "SOURCE_CHANGED", "Another upload reserved this version with different source files", @@ -134,21 +151,13 @@ export async function buildStoredPackage(input: { sourceId: source.id, variants }); - const objectId = await reserveObject( - repository, - store, - owner, - manifest.id, - built.bytes, - built.packageSha256 - ); const artifactId = crypto.randomUUID(); const downloadUrl = `${input.baseUrl}/packages/${artifactId}/download`; const artifact = await repository.linkArtifact(owner, manifest, { id: artifactId, sourceId: source.id, manifestHash, - sha256: built.packageSha256, + sha256: built.archive.sha256, objectId, downloadUrl }); @@ -162,6 +171,10 @@ export async function buildStoredPackage(input: { await store.delete(objectId); await repository.removeObject(objectId); } + for (const id of iconIds ?? []) { + await store.delete(id); + await repository.removeObject(id); + } return artifact; } diff --git a/src/services/extensions/v2/packages/jobs.ts b/services/api/src/services/extensions/v2/packages/jobs.ts similarity index 80% rename from src/services/extensions/v2/packages/jobs.ts rename to services/api/src/services/extensions/v2/packages/jobs.ts index 0af883d0..816504d5 100644 --- a/src/services/extensions/v2/packages/jobs.ts +++ b/services/api/src/services/extensions/v2/packages/jobs.ts @@ -1,10 +1,10 @@ -import { logError } from "../../../../lib/logger"; +import { logError } from "@fossbilling/logging"; import type { IMessageQueue } from "../../../../lib/interfaces"; import type { Manifest } from "../schemas/manifest"; import type { PackageOwner } from "./repository"; -import { PackageError } from "./errors"; +import { PackageError } from "@fossbilling/extension-packaging/errors"; -export interface PackageJob { +export interface PackagingJob { id: string; fingerprint: string; owner: PackageOwner; @@ -20,22 +20,22 @@ export interface PackageJob { updatedAt: string; } -export interface PackageJobRepository { - get(id: string): Promise; - claim(id: string, lease: string): Promise; +export interface PackagingJobRepository { + get(id: string): Promise; + claim(id: string, lease: string): Promise; complete(id: string, lease: string, artifactId: string): Promise; fail( id: string, lease: string, - error: NonNullable, + error: NonNullable, retry: boolean ): Promise; pending(): Promise; markEnqueued(id: string): Promise; } -export async function dispatchPackageJob( - jobs: PackageJobRepository, +export async function dispatchPackagingJob( + jobs: PackagingJobRepository, queue: IMessageQueue, id: string ) { @@ -44,10 +44,10 @@ export async function dispatchPackageJob( } // Delivery-specific acknowledgement/retry APIs stay in the platform adapter. -export async function processPackageJob( - jobs: PackageJobRepository, +export async function processPackagingJob( + jobs: PackagingJobRepository, id: string, - prepare: (job: PackageJob) => Promise, + prepare: (job: PackagingJob) => Promise, lease = crypto.randomUUID() ): Promise<"ack" | "retry" | "busy"> { const existing = await jobs.get(id); diff --git a/src/services/extensions/v2/packages/manifest.ts b/services/api/src/services/extensions/v2/packages/manifest.ts similarity index 100% rename from src/services/extensions/v2/packages/manifest.ts rename to services/api/src/services/extensions/v2/packages/manifest.ts diff --git a/src/services/extensions/v2/packages/prepare.ts b/services/api/src/services/extensions/v2/packages/prepare.ts similarity index 59% rename from src/services/extensions/v2/packages/prepare.ts rename to services/api/src/services/extensions/v2/packages/prepare.ts index 35061266..c924f8e3 100644 --- a/src/services/extensions/v2/packages/prepare.ts +++ b/services/api/src/services/extensions/v2/packages/prepare.ts @@ -1,22 +1,20 @@ import type { PackageRepository } from "./repository"; -import type { PackageImageProcessor } from "./images"; -import type { PackageStore } from "./store"; -import type { PackageJob } from "./jobs"; -import { PackageError } from "./errors"; -import { boundedBytes } from "./store"; +import type { PackageProcessor } from "@fossbilling/extension-packaging/processor"; +import type { PackageStore } from "@fossbilling/extension-packaging/store"; +import type { PackagingJob } from "./jobs"; +import { PackageError } from "@fossbilling/extension-packaging/errors"; import { buildStoredPackage } from "./build"; -import { PACKAGE_LIMITS } from "../package-builder"; export interface PackagePreparation { repository: PackageRepository; store: PackageStore; - images: PackageImageProcessor; + processor: PackageProcessor; baseUrl: string; } export async function prepareQueuedPackage( services: PackagePreparation, - job: PackageJob + job: PackagingJob ): Promise { const { repository, store } = services; const previous = job.owner; @@ -41,23 +39,10 @@ export async function prepareQueuedPackage( "Package ownership changed while queued", 403 ); - let upload: { bytes: Uint8Array; objectId: string } | undefined; - if (job.uploadObjectId) { - const body = await store.get( - job.uploadObjectId, - job.uploadSha256 ?? undefined - ); - if (!body) - throw new PackageError( - "SOURCE_UNAVAILABLE", - "Queued source ZIP is unavailable", - 503 - ); - upload = { - bytes: await boundedBytes(body, PACKAGE_LIMITS.sourceBytes), - objectId: job.uploadObjectId - }; - } + const upload = + job.uploadObjectId && job.uploadSha256 + ? { objectId: job.uploadObjectId, sha256: job.uploadSha256 } + : undefined; const artifact = await buildStoredPackage({ repository, store, @@ -66,7 +51,7 @@ export async function prepareQueuedPackage( upload, sourceId: job.sourceId ?? undefined, baseUrl: services.baseUrl.replace(/\/$/, ""), - images: services.images + processor: services.processor }); return artifact.id; } diff --git a/services/api/src/services/extensions/v2/packages/processor.ts b/services/api/src/services/extensions/v2/packages/processor.ts new file mode 100644 index 00000000..e895216a --- /dev/null +++ b/services/api/src/services/extensions/v2/packages/processor.ts @@ -0,0 +1,20 @@ +import { HttpPackageProcessor } from "@fossbilling/extension-packaging/adapters/http"; +import { PackageError } from "@fossbilling/extension-packaging/errors"; + +export function configuredPackageProcessor( + getEnv: (name: string) => string | undefined +) { + const url = getEnv("EXTENSION_PACKAGER_URL"); + const secret = getEnv("EXTENSION_PACKAGER_SECRET"); + if (!url || !secret) + throw new PackageError( + "PACKAGES_DISABLED", + "Package processor is not configured", + 503 + ); + return new HttpPackageProcessor( + url, + secret, + getEnv("ENVIRONMENT") !== "production" + ); +} diff --git a/src/services/extensions/v2/packages/repository.ts b/services/api/src/services/extensions/v2/packages/repository.ts similarity index 97% rename from src/services/extensions/v2/packages/repository.ts rename to services/api/src/services/extensions/v2/packages/repository.ts index a43e097c..b3d25488 100644 --- a/src/services/extensions/v2/packages/repository.ts +++ b/services/api/src/services/extensions/v2/packages/repository.ts @@ -65,6 +65,7 @@ export interface PackageRepository { id: string, bytes: number ): Promise; + setObjectSize(id: string, bytes: number): Promise; removeObject(id: string): Promise; claimCleanup(): Promise; finishCleanup(id: string): Promise; diff --git a/src/lib/adapters/cloudflare/package-store.ts b/services/api/src/services/extensions/v2/packages/store.ts similarity index 74% rename from src/lib/adapters/cloudflare/package-store.ts rename to services/api/src/services/extensions/v2/packages/store.ts index cfc073a8..71ccd8da 100644 --- a/src/lib/adapters/cloudflare/package-store.ts +++ b/services/api/src/services/extensions/v2/packages/store.ts @@ -1,17 +1,14 @@ -import { PackageError } from "../../../services/extensions/v2/packages/errors"; -import type { PackageStore } from "../../../services/extensions/v2/packages/store"; +import { PackageError } from "@fossbilling/extension-packaging/errors"; +import type { PackageStore } from "@fossbilling/extension-packaging/store"; import type { Context } from "hono"; -import { b2ObjectStore } from "../b2/package-store"; -import { getPlatform } from "../../middleware"; +import { b2ObjectStore } from "@fossbilling/extension-packaging/adapters/b2"; +import { getPlatform } from "../../../../lib/middleware"; -export function packageStore( - c: Context<{ Bindings: CloudflareBindings }> -): PackageStore | null { - return configuredPackageStore(c.env, (name) => getPlatform(c).getEnv(name)); +export function packageStore(c: Context): PackageStore | null { + return configuredPackageStore((name) => getPlatform(c).getEnv(name)); } export function configuredPackageStore( - env: CloudflareBindings, getEnv: (name: string) => string | undefined ): PackageStore | null { const mode = getEnv("EXTENSION_PACKAGES_MODE"); @@ -40,8 +37,7 @@ export function configuredPackageStore( !getEnv("EXTENSION_PACKAGES_S3_PREFIX") || getEnv("EXTENSION_PACKAGES_S3_PREFIX") === "local" || !getEnv("EXTENSION_ASSETS_S3_ENDPOINT") || - !getEnv("EXTENSION_ASSETS_PUBLIC_URL") || - !env.PACKAGE_IMAGES + !getEnv("EXTENSION_ASSETS_PUBLIC_URL") ) { throw new PackageError( "PACKAGES_DISABLED", diff --git a/src/services/extensions/v2/resource-limits.ts b/services/api/src/services/extensions/v2/resource-limits.ts similarity index 100% rename from src/services/extensions/v2/resource-limits.ts rename to services/api/src/services/extensions/v2/resource-limits.ts diff --git a/src/services/extensions/v2/revalidate.ts b/services/api/src/services/extensions/v2/revalidate.ts similarity index 73% rename from src/services/extensions/v2/revalidate.ts rename to services/api/src/services/extensions/v2/revalidate.ts index ffa6cacd..6e1a5b43 100644 --- a/src/services/extensions/v2/revalidate.ts +++ b/services/api/src/services/extensions/v2/revalidate.ts @@ -19,26 +19,29 @@ import { invalidateListCache } from "../v1/list-cache"; const REVALIDATE_TAGS = ["extensions", "developers"]; const REVALIDATE_URL = "https://extensions.fossbilling.org/api/revalidate"; -export function revalidateExtensions( - c: Context<{ Bindings: CloudflareBindings }> -): void { - invalidateListCache(c.env.CACHE_KV, (p) => c.executionCtx.waitUntil(p)); +export function revalidateExtensions(c: Context): void { + const platform = getPlatform(c); + invalidateListCache(platform.getCache("CACHE_KV"), platform.waitUntil); - const secret = getPlatform(c).getEnv("EXTENSIONS_REVALIDATE_SECRET"); + const secret = platform.getEnv("EXTENSIONS_REVALIDATE_SECRET"); if (!secret) { // Unconfigured (e.g. local dev without the secret): skipping is safe — // the site's cache windows bound staleness regardless. return; } - const purge = c.env.EXTENSIONS_FRONTEND.fetch(REVALIDATE_URL, { - method: "POST", - headers: { - authorization: `Bearer ${secret}`, - "content-type": "application/json" - }, - body: JSON.stringify({ tags: REVALIDATE_TAGS }) - }) + const purge = platform + .fetchService( + "EXTENSIONS_FRONTEND", + new Request(REVALIDATE_URL, { + method: "POST", + headers: { + authorization: `Bearer ${secret}`, + "content-type": "application/json" + }, + body: JSON.stringify({ tags: REVALIDATE_TAGS }) + }) + ) .then(async (response) => { if (!response.ok) { console.error( @@ -50,5 +53,5 @@ export function revalidateExtensions( console.error("[revalidate] purge request failed:", error); }); - c.executionCtx.waitUntil(purge); + platform.waitUntil(purge); } diff --git a/src/services/extensions/v2/routes/account.ts b/services/api/src/services/extensions/v2/routes/account.ts similarity index 99% rename from src/services/extensions/v2/routes/account.ts rename to services/api/src/services/extensions/v2/routes/account.ts index cc30b416..7a41dc4b 100644 --- a/src/services/extensions/v2/routes/account.ts +++ b/services/api/src/services/extensions/v2/routes/account.ts @@ -3,7 +3,7 @@ import { requireIdentitySync, requireUser } from "../middleware"; -import { getAuth } from "../../../../lib/auth"; +import { getAuth } from "../../../../lib/auth/index"; import { getExtensionsDb } from "../../../../lib/db"; import { createRoute, z } from "@hono/zod-openapi"; import { errorBody, statusFromErrorCode } from "./errors"; diff --git a/src/services/extensions/v2/routes/app.ts b/services/api/src/services/extensions/v2/routes/app.ts similarity index 100% rename from src/services/extensions/v2/routes/app.ts rename to services/api/src/services/extensions/v2/routes/app.ts diff --git a/src/services/extensions/v2/routes/badges.ts b/services/api/src/services/extensions/v2/routes/badges.ts similarity index 100% rename from src/services/extensions/v2/routes/badges.ts rename to services/api/src/services/extensions/v2/routes/badges.ts diff --git a/src/services/extensions/v2/routes/developer-profiles.ts b/services/api/src/services/extensions/v2/routes/developer-profiles.ts similarity index 97% rename from src/services/extensions/v2/routes/developer-profiles.ts rename to services/api/src/services/extensions/v2/routes/developer-profiles.ts index 4a76dbad..216d3386 100644 --- a/src/services/extensions/v2/routes/developer-profiles.ts +++ b/services/api/src/services/extensions/v2/routes/developer-profiles.ts @@ -6,7 +6,7 @@ import { } from "../middleware"; import { getExtensionsDb } from "../../../../lib/db"; import { getPlatform } from "../../../../lib/middleware"; -import { getAuth } from "../../../../lib/auth"; +import { getAuth } from "../../../../lib/auth/index"; import { createRoute, z } from "@hono/zod-openapi"; import { errorBody, @@ -195,16 +195,11 @@ export function registerDeveloperProfileRoutes(app: ExtensionsV2App): void { auth.userId, body, platform.getEnv("GITHUB_TOKEN"), - // Cloudflare enforces the configured 3-per-60s account allowance across - // isolates. Keep this as a callback so upsertOwn can run its cheap + // Keep this as a callback so upsertOwn can run its cheap // existing-profile/id checks first; updates and known-taken ids must not // spend creation allowance. - async () => - ( - await c.env.PROFILE_CREATION_RATE_LIMITER.limit({ - key: auth.userId - }) - ).success + () => + platform.checkRateLimit("PROFILE_CREATION_RATE_LIMITER", auth.userId) ); if (error || !data) { const status = diff --git a/src/services/extensions/v2/routes/errors.ts b/services/api/src/services/extensions/v2/routes/errors.ts similarity index 100% rename from src/services/extensions/v2/routes/errors.ts rename to services/api/src/services/extensions/v2/routes/errors.ts diff --git a/src/services/extensions/v2/routes/m2m.ts b/services/api/src/services/extensions/v2/routes/m2m.ts similarity index 100% rename from src/services/extensions/v2/routes/m2m.ts rename to services/api/src/services/extensions/v2/routes/m2m.ts diff --git a/src/services/extensions/v2/routes/moderation.ts b/services/api/src/services/extensions/v2/routes/moderation.ts similarity index 99% rename from src/services/extensions/v2/routes/moderation.ts rename to services/api/src/services/extensions/v2/routes/moderation.ts index ebdde3a3..6de0a685 100644 --- a/src/services/extensions/v2/routes/moderation.ts +++ b/services/api/src/services/extensions/v2/routes/moderation.ts @@ -6,7 +6,7 @@ import { getPublisher } from "../middleware"; import { getExtensionsDb } from "../../../../lib/db"; -import { getAuth } from "../../../../lib/auth"; +import { getAuth } from "../../../../lib/auth/index"; import { createRoute, z } from "@hono/zod-openapi"; import { errorBody, diff --git a/src/services/extensions/v2/routes/notify.ts b/services/api/src/services/extensions/v2/routes/notify.ts similarity index 91% rename from src/services/extensions/v2/routes/notify.ts rename to services/api/src/services/extensions/v2/routes/notify.ts index f5c4706f..61602f4a 100644 --- a/src/services/extensions/v2/routes/notify.ts +++ b/services/api/src/services/extensions/v2/routes/notify.ts @@ -26,7 +26,6 @@ export async function notifyAuthor( ): Promise { revalidateExtensions(c); if (!notifyRequested(query)) return false; - return sendModerationNotification(getPlatform(c), extDb, input, (p) => - c.executionCtx.waitUntil(p) - ); + const platform = getPlatform(c); + return sendModerationNotification(platform, extDb, input, platform.waitUntil); } diff --git a/src/services/extensions/v2/routes/owner-extensions.ts b/services/api/src/services/extensions/v2/routes/owner-extensions.ts similarity index 99% rename from src/services/extensions/v2/routes/owner-extensions.ts rename to services/api/src/services/extensions/v2/routes/owner-extensions.ts index dfd786cb..845c6dcf 100644 --- a/src/services/extensions/v2/routes/owner-extensions.ts +++ b/services/api/src/services/extensions/v2/routes/owner-extensions.ts @@ -16,7 +16,7 @@ import { getPublisher } from "../middleware"; import { getExtensionsDb } from "../../../../lib/db"; -import { getAuth } from "../../../../lib/auth"; +import { getAuth } from "../../../../lib/auth/index"; import { createRoute, z } from "@hono/zod-openapi"; import { ActiveAccountRequiredResponse, diff --git a/src/services/extensions/v2/routes/ownership.ts b/services/api/src/services/extensions/v2/routes/ownership.ts similarity index 99% rename from src/services/extensions/v2/routes/ownership.ts rename to services/api/src/services/extensions/v2/routes/ownership.ts index 587386b7..e3f8b8c4 100644 --- a/src/services/extensions/v2/routes/ownership.ts +++ b/services/api/src/services/extensions/v2/routes/ownership.ts @@ -1,7 +1,7 @@ import { requireActiveAuth, requireModerator } from "../middleware"; import { getExtensionsDb } from "../../../../lib/db"; import { getPlatform } from "../../../../lib/middleware"; -import { getAuth } from "../../../../lib/auth"; +import { getAuth } from "../../../../lib/auth/index"; import { createRoute, z } from "@hono/zod-openapi"; import { errorBody, diff --git a/src/services/extensions/v2/routes/packages.ts b/services/api/src/services/extensions/v2/routes/packages.ts similarity index 90% rename from src/services/extensions/v2/routes/packages.ts rename to services/api/src/services/extensions/v2/routes/packages.ts index 1cdacdd9..2da74192 100644 --- a/src/services/extensions/v2/routes/packages.ts +++ b/services/api/src/services/extensions/v2/routes/packages.ts @@ -1,9 +1,10 @@ -import { PackageJobsDatabase } from "../db/package-jobs"; -import { dispatchPackageJob, type PackageJob } from "../packages/jobs"; -import { sha256 } from "../package-builder"; +import { configuredPackageProcessor } from "../packages/processor"; +import { PackagingJobsDatabase } from "../db/packaging-jobs"; +import { dispatchPackagingJob, type PackagingJob } from "../packages/jobs"; +import { sha256 } from "@fossbilling/extension-packaging/checksum"; import { createRoute, z } from "@hono/zod-openapi"; import type { Context, MiddlewareHandler } from "hono"; -import { getAuth } from "../../../../lib/auth"; +import { getAuth } from "../../../../lib/auth/index"; import { getPlatform } from "../../../../lib/middleware"; import { canonicalJson } from "../packages/manifest"; import type { ArtifactRow } from "../packages/repository"; @@ -18,14 +19,14 @@ import { paceContentIp } from "../middleware"; import { reserveObject, cleanupPackageObjects } from "../packages/build"; -import { PackageError } from "../packages/errors"; -import { boundedBytes } from "../packages/store"; -import { packageStore } from "../../../../lib/adapters/cloudflare/package-store"; +import { PackageError } from "@fossbilling/extension-packaging/errors"; +import { boundedBytes } from "@fossbilling/extension-packaging/store"; +import { packageStore } from "../packages/store"; import { ManifestSchema } from "../schemas/manifest"; import { PackageArtifactSchema, - PackageJobSchema, - PackageJobParamSchema, + PackagingJobSchema, + PackagingJobParamSchema, PackageBuildSchema, PackageParamSchema } from "../schemas/packages"; @@ -62,7 +63,7 @@ function artifactResponse(row: ArtifactRow) { } }; } -async function jobResponse(db: D1Database, job: PackageJob) { +async function jobResponse(db: D1Database, job: PackagingJob) { const artifact = job.artifactId ? await new PackageBuildDatabase(db).getArtifact(job.artifactId) : null; @@ -111,8 +112,20 @@ export function registerPackageRoutes(app: ExtensionsV2App) { }; app.use("/packages", availability); app.use("/packages/*", availability); - app.use("/package-jobs/*", availability); - app.on("POST", "/packages", paceContentIp()); + app.use("/packaging-jobs/*", availability); + app.on( + "POST", + "/packages", + async (c, next) => { + try { + configuredPackageProcessor(getPlatform(c).getEnv); + } catch (error) { + return failure(c, error); + } + await next(); + }, + paceContentIp() + ); const boundUpload: MiddlewareHandler = async (c, next) => { try { const body = await boundedBytes(c.req.raw.body, 6 * 1024 * 1024); @@ -148,7 +161,7 @@ export function registerPackageRoutes(app: ExtensionsV2App) { "Private processing job; identical requests reuse queued or completed work", content: { "application/json": { - schema: z.object({ result: PackageJobSchema }) + schema: z.object({ result: PackagingJobSchema }) } } }, @@ -208,7 +221,7 @@ export function registerPackageRoutes(app: ExtensionsV2App) { "Package download URL is not configured", 503 ); - const jobs = new PackageJobsDatabase(db); + const jobs = new PackagingJobsDatabase(db); const source = form.source ? new Uint8Array(await form.source.arrayBuffer()) : undefined; @@ -246,14 +259,14 @@ export function registerPackageRoutes(app: ExtensionsV2App) { }); } if (job.status === "queued") - await dispatchPackageJob( + await dispatchPackagingJob( jobs, getPlatform(c).getQueue("EXTENSION_PACKAGE_QUEUE"), job.id ); c.header( "Location", - `${baseUrl.replace(/\/$/, "")}/package-jobs/${job.id}` + `${baseUrl.replace(/\/$/, "")}/packaging-jobs/${job.id}` ); c.header("Retry-After", "1"); return c.json(await jobResponse(db, job), 202); @@ -266,18 +279,18 @@ export function registerPackageRoutes(app: ExtensionsV2App) { app.openapi( createRoute({ method: "get", - path: "/package-jobs/{jobId}", + path: "/packaging-jobs/{jobId}", tags: ["Packages"], - summary: "Read an owning developer's private package-processing job", + summary: "Read an owning developer's private packaging job", security: [{ Bearer: [] }], middleware: [requireContentWrite()] as const, - request: { params: PackageJobParamSchema }, + request: { params: PackagingJobParamSchema }, responses: { 200: { description: "Package processing state and completed artifact", content: { "application/json": { - schema: z.object({ result: PackageJobSchema }) + schema: z.object({ result: PackagingJobSchema }) } } }, @@ -288,11 +301,11 @@ export function registerPackageRoutes(app: ExtensionsV2App) { try { const db = c.env.DB_EXTENSIONS; const repository = new PackageBuildDatabase(db); - const job = await new PackageJobsDatabase(db).get( + const job = await new PackagingJobsDatabase(db).get( c.req.valid("param").jobId ); if (!job) - throw new PackageError("NOT_FOUND", "Package job not found", 404); + throw new PackageError("NOT_FOUND", "Packaging job not found", 404); const userId = getAuth(c).userId; const moderator = await repository.isModerator(userId); if (!moderator) { @@ -308,7 +321,7 @@ export function registerPackageRoutes(app: ExtensionsV2App) { owner.ownership_epoch !== job.owner.ownership_epoch || (publisher && publisher.clientId !== job.owner.publisher_client_id) ) - throw new PackageError("NOT_FOUND", "Package job not found", 404); + throw new PackageError("NOT_FOUND", "Packaging job not found", 404); } return c.json(await jobResponse(db, job), 200); } catch (error) { @@ -341,7 +354,7 @@ export function registerPackageRoutes(app: ExtensionsV2App) { }), async (c) => { try { - await new PackageJobsDatabase(c.env.DB_EXTENSIONS).expire(); + await new PackagingJobsDatabase(c.env.DB_EXTENSIONS).expire(); return c.json( { result: { diff --git a/src/services/extensions/v2/routes/public-extensions.ts b/services/api/src/services/extensions/v2/routes/public-extensions.ts similarity index 100% rename from src/services/extensions/v2/routes/public-extensions.ts rename to services/api/src/services/extensions/v2/routes/public-extensions.ts diff --git a/src/services/extensions/v2/routes/revisions.ts b/services/api/src/services/extensions/v2/routes/revisions.ts similarity index 97% rename from src/services/extensions/v2/routes/revisions.ts rename to services/api/src/services/extensions/v2/routes/revisions.ts index 2222d613..2f86d7a8 100644 --- a/src/services/extensions/v2/routes/revisions.ts +++ b/services/api/src/services/extensions/v2/routes/revisions.ts @@ -1,4 +1,4 @@ -import { getAuth } from "../../../../lib/auth"; +import { getAuth } from "../../../../lib/auth/index"; import { getExtensionsDb } from "../../../../lib/db"; import { createRoute, z } from "@hono/zod-openapi"; import { errorBody, listErrorStatus, listPayload } from "./errors"; diff --git a/src/services/extensions/v2/schemas/badges.ts b/services/api/src/services/extensions/v2/schemas/badges.ts similarity index 100% rename from src/services/extensions/v2/schemas/badges.ts rename to services/api/src/services/extensions/v2/schemas/badges.ts diff --git a/src/services/extensions/v2/schemas/common.ts b/services/api/src/services/extensions/v2/schemas/common.ts similarity index 94% rename from src/services/extensions/v2/schemas/common.ts rename to services/api/src/services/extensions/v2/schemas/common.ts index c709a03d..6ba3ea40 100644 --- a/src/services/extensions/v2/schemas/common.ts +++ b/services/api/src/services/extensions/v2/schemas/common.ts @@ -8,18 +8,6 @@ export const lowercaseId = (label: string) => message: `${label} id must be a lowercase alphanumeric slug` }); -// Restricts to http(s) — z.string().url() alone accepts any scheme, -// including javascript:/data:, which is unsafe for fields a consumer may -// render as a link or image src. -export const httpUrl = () => - z - .string() - .max(2048) - .url() - .refine((value) => /^https?:\/\//i.test(value), { - message: "must use http or https" - }); - export const ErrorResponseSchema = z .object({ error: z.object({ diff --git a/src/services/extensions/v2/schemas/developers.ts b/services/api/src/services/extensions/v2/schemas/developers.ts similarity index 98% rename from src/services/extensions/v2/schemas/developers.ts rename to services/api/src/services/extensions/v2/schemas/developers.ts index bd4af04a..13ac32b1 100644 --- a/src/services/extensions/v2/schemas/developers.ts +++ b/services/api/src/services/extensions/v2/schemas/developers.ts @@ -1,5 +1,6 @@ import { z } from "@hono/zod-openapi"; -import { httpUrl, lowercaseId, CursorPaginationQuerySchema } from "./common"; +import { httpUrl } from "@fossbilling/extension-packaging/urls"; +import { lowercaseId, CursorPaginationQuerySchema } from "./common"; // GET /developers/unapproved has been merged into GET /developers?scope=. // "unapproved" is therefore no longer a shadowed static route, but stays diff --git a/src/services/extensions/v2/schemas/extensions.ts b/services/api/src/services/extensions/v2/schemas/extensions.ts similarity index 97% rename from src/services/extensions/v2/schemas/extensions.ts rename to services/api/src/services/extensions/v2/schemas/extensions.ts index be505f97..ddb6ea18 100644 --- a/src/services/extensions/v2/schemas/extensions.ts +++ b/services/api/src/services/extensions/v2/schemas/extensions.ts @@ -1,11 +1,9 @@ +import { MANIFEST_EXTENSION_TYPES } from "@fossbilling/extension-packaging/manifest"; import { MAX_CONTENT_BYTES } from "../resource-limits"; import { z } from "@hono/zod-openapi"; -import { - ManifestSchema, - MANIFEST_EXTENSION_TYPES, - ManifestRequirementsSchema -} from "./manifest"; -import { cursorPaginationFields, httpUrl, PaginationSchema } from "./common"; +import { ManifestSchema, ManifestRequirementsSchema } from "./manifest"; +import { cursorPaginationFields, PaginationSchema } from "./common"; +import { httpUrl } from "@fossbilling/extension-packaging/urls"; import { PublicDeveloperSchema } from "./developers"; export const EXTENSION_TYPES = MANIFEST_EXTENSION_TYPES; diff --git a/src/services/extensions/v2/schemas/m2m.ts b/services/api/src/services/extensions/v2/schemas/m2m.ts similarity index 100% rename from src/services/extensions/v2/schemas/m2m.ts rename to services/api/src/services/extensions/v2/schemas/m2m.ts diff --git a/services/api/src/services/extensions/v2/schemas/manifest.ts b/services/api/src/services/extensions/v2/schemas/manifest.ts new file mode 100644 index 00000000..aa35b320 --- /dev/null +++ b/services/api/src/services/extensions/v2/schemas/manifest.ts @@ -0,0 +1,26 @@ +import { z } from "@hono/zod-openapi"; +import { + ManifestSchema as PackageManifestSchema, + ManifestRequirementsSchema as PackageRequirementsSchema +} from "@fossbilling/extension-packaging/manifest"; + +export const ManifestRequirementsSchema = z + .strictObject(PackageRequirementsSchema.shape) + .openapi("ManifestRequirements"); +export const ManifestSchema = z + .strictObject({ + ...PackageManifestSchema.shape, + requires: ManifestRequirementsSchema + }) + .openapi("Manifest"); +export type Manifest = z.infer; + +export function manifestMetadata( + content: Pick +) { + return { + manifest_version: content.manifest_version, + author: content.author, + requires: content.requires + }; +} diff --git a/src/services/extensions/v2/schemas/ownership.ts b/services/api/src/services/extensions/v2/schemas/ownership.ts similarity index 100% rename from src/services/extensions/v2/schemas/ownership.ts rename to services/api/src/services/extensions/v2/schemas/ownership.ts diff --git a/src/services/extensions/v2/schemas/packages.ts b/services/api/src/services/extensions/v2/schemas/packages.ts similarity index 90% rename from src/services/extensions/v2/schemas/packages.ts rename to services/api/src/services/extensions/v2/schemas/packages.ts index 64f90d40..c76c3172 100644 --- a/src/services/extensions/v2/schemas/packages.ts +++ b/services/api/src/services/extensions/v2/schemas/packages.ts @@ -32,8 +32,8 @@ export const PackageArtifactSchema = z }) .openapi("PackageArtifact"); -export const PackageJobParamSchema = z.object({ jobId: z.uuid() }); -export const PackageJobSchema = z +export const PackagingJobParamSchema = z.object({ jobId: z.uuid() }); +export const PackagingJobSchema = z .object({ id: z.uuid(), extension_identifier: z.string(), @@ -49,4 +49,4 @@ export const PackageJobSchema = z created_at: z.string(), updated_at: z.string() }) - .openapi("PackageJob"); + .openapi("PackagingJob"); diff --git a/src/services/extensions/v2/schemas/revisions.ts b/services/api/src/services/extensions/v2/schemas/revisions.ts similarity index 100% rename from src/services/extensions/v2/schemas/revisions.ts rename to services/api/src/services/extensions/v2/schemas/revisions.ts diff --git a/src/services/extensions/v2/schemas/users.ts b/services/api/src/services/extensions/v2/schemas/users.ts similarity index 100% rename from src/services/extensions/v2/schemas/users.ts rename to services/api/src/services/extensions/v2/schemas/users.ts diff --git a/src/services/previews/v1/README.md b/services/api/src/services/previews/v1/README.md similarity index 100% rename from src/services/previews/v1/README.md rename to services/api/src/services/previews/v1/README.md diff --git a/src/services/previews/v1/budget.ts b/services/api/src/services/previews/v1/budget.ts similarity index 100% rename from src/services/previews/v1/budget.ts rename to services/api/src/services/previews/v1/budget.ts diff --git a/src/services/previews/v1/cache.ts b/services/api/src/services/previews/v1/cache.ts similarity index 98% rename from src/services/previews/v1/cache.ts rename to services/api/src/services/previews/v1/cache.ts index a340434a..561c09f5 100644 --- a/src/services/previews/v1/cache.ts +++ b/services/api/src/services/previews/v1/cache.ts @@ -1,3 +1,4 @@ +import type { ICache } from "../../../lib/interfaces"; import { GithubLookupResult } from "./github/artifacts"; import { singleFlight } from "../../../lib/cache"; @@ -33,7 +34,7 @@ export type CachedRead = { status: "hit"; value: T } | { status: "negative" } | { status: "miss" }; export async function readCachedValue( - kv: KVNamespace, + kv: ICache, key: string ): Promise> { const cached = await kv.get(key); @@ -59,7 +60,7 @@ export async function readCachedValue( // waitUntil, when provided, receives the KV writes so they don't sit on // the response path. export async function cachedLookup( - kv: KVNamespace, + kv: ICache, key: string, resolve: () => Promise>, ttlSeconds: number | ((data: T) => number) = DEFAULT_CACHE_TTL_SECONDS, diff --git a/src/services/previews/v1/github/artifacts.ts b/services/api/src/services/previews/v1/github/artifacts.ts similarity index 99% rename from src/services/previews/v1/github/artifacts.ts rename to services/api/src/services/previews/v1/github/artifacts.ts index 27dfd6c2..24818d5d 100644 --- a/src/services/previews/v1/github/artifacts.ts +++ b/services/api/src/services/previews/v1/github/artifacts.ts @@ -4,7 +4,7 @@ import { GitHubError, NotFoundError } from "../../../../lib/github-errors"; -import { logWarn } from "../../../../lib/logger"; +import { logWarn } from "@fossbilling/logging"; const REPO_OWNER = "FOSSBilling"; const REPO_NAME = "FOSSBilling"; diff --git a/src/services/previews/v1/github/request.ts b/services/api/src/services/previews/v1/github/request.ts similarity index 100% rename from src/services/previews/v1/github/request.ts rename to services/api/src/services/previews/v1/github/request.ts diff --git a/src/services/previews/v1/index.ts b/services/api/src/services/previews/v1/index.ts similarity index 100% rename from src/services/previews/v1/index.ts rename to services/api/src/services/previews/v1/index.ts diff --git a/src/services/previews/v1/r2.ts b/services/api/src/services/previews/v1/r2.ts similarity index 100% rename from src/services/previews/v1/r2.ts rename to services/api/src/services/previews/v1/r2.ts diff --git a/src/services/previews/v1/resolve.ts b/services/api/src/services/previews/v1/resolve.ts similarity index 100% rename from src/services/previews/v1/resolve.ts rename to services/api/src/services/previews/v1/resolve.ts diff --git a/src/services/previews/v1/routes/app.ts b/services/api/src/services/previews/v1/routes/app.ts similarity index 100% rename from src/services/previews/v1/routes/app.ts rename to services/api/src/services/previews/v1/routes/app.ts diff --git a/src/services/previews/v1/routes/commit.ts b/services/api/src/services/previews/v1/routes/commit.ts similarity index 96% rename from src/services/previews/v1/routes/commit.ts rename to services/api/src/services/previews/v1/routes/commit.ts index f549b076..2172e126 100644 --- a/src/services/previews/v1/routes/commit.ts +++ b/services/api/src/services/previews/v1/routes/commit.ts @@ -1,3 +1,4 @@ +import { getPlatform } from "../../../../lib/middleware"; import { previewGitHub } from "../github/request"; import { createRoute } from "@hono/zod-openapi"; import { @@ -92,11 +93,11 @@ export function registerCommitRoutes(app: PreviewsV1App): void { resolveArtifactPreview(github, sha, null) ) : await cachedLookup( - c.env.CACHE_KV, + getPlatform(c).getCache("CACHE_KV"), cacheKeyForSha(sha), () => resolveArtifactPreview(github, sha, null), ttlForArtifact, - (p) => c.executionCtx.waitUntil(p) + getPlatform(c).waitUntil ); return respondWithLookup( @@ -137,11 +138,11 @@ export function registerCommitRoutes(app: PreviewsV1App): void { resolveArtifactPreview(github, sha, null) ) : await cachedLookup( - c.env.CACHE_KV, + getPlatform(c).getCache("CACHE_KV"), cacheKeyForSha(sha), () => resolveArtifactPreview(github, sha, null), ttlForArtifact, - (p) => c.executionCtx.waitUntil(p) + getPlatform(c).waitUntil ); return respondWithDownloadRedirect( c, diff --git a/src/services/previews/v1/routes/errors.ts b/services/api/src/services/previews/v1/routes/errors.ts similarity index 100% rename from src/services/previews/v1/routes/errors.ts rename to services/api/src/services/previews/v1/routes/errors.ts diff --git a/src/services/previews/v1/routes/main.ts b/services/api/src/services/previews/v1/routes/main.ts similarity index 97% rename from src/services/previews/v1/routes/main.ts rename to services/api/src/services/previews/v1/routes/main.ts index e0b23d3d..b216de60 100644 --- a/src/services/previews/v1/routes/main.ts +++ b/services/api/src/services/previews/v1/routes/main.ts @@ -1,3 +1,4 @@ +import { getPlatform } from "../../../../lib/middleware"; import { PreviewGitHub, previewGitHub } from "../github/request"; import { createRoute } from "@hono/zod-openapi"; import { Context } from "hono"; @@ -92,7 +93,7 @@ async function resolveMainPreview( c: Context<{ Bindings: CloudflareBindings }> ): Promise { const result = await cachedLookup( - c.env.CACHE_KV, + getPlatform(c).getCache("CACHE_KV"), MAIN_CACHE_KEY, async () => { const object = await resolveMainObject(c); @@ -114,7 +115,7 @@ async function resolveMainPreview( }; }, MAIN_CACHE_TTL_SECONDS, - (p) => c.executionCtx.waitUntil(p) + getPlatform(c).waitUntil ); return result.status === "found" ? result.data : null; } @@ -124,7 +125,7 @@ async function resolveMainDownloadUrl( c: Context<{ Bindings: CloudflareBindings }> ): Promise { const read = await readCachedValue( - c.env.CACHE_KV, + getPlatform(c).getCache("CACHE_KV"), MAIN_CACHE_KEY ); if (read.status === "negative") return null; // authoritative negative diff --git a/src/services/previews/v1/routes/pr.ts b/services/api/src/services/previews/v1/routes/pr.ts similarity index 88% rename from src/services/previews/v1/routes/pr.ts rename to services/api/src/services/previews/v1/routes/pr.ts index f105d7dd..ece9eb26 100644 --- a/src/services/previews/v1/routes/pr.ts +++ b/services/api/src/services/previews/v1/routes/pr.ts @@ -1,3 +1,5 @@ +import type { ICache } from "../../../../lib/interfaces"; +import { getPlatform } from "../../../../lib/middleware"; import { PreviewGitHub, previewGitHub } from "../github/request"; import { createRoute } from "@hono/zod-openapi"; import { @@ -22,7 +24,7 @@ import { PreviewsV1App } from "./app"; async function resolvePrPreview( github: PreviewGitHub, prNumber: number, - kv: KVNamespace, + kv: ICache, waitUntil?: (promise: Promise) => void ): Promise { const head = await resolvePullRequestHeadSha(github, prNumber); @@ -72,14 +74,17 @@ export function registerPrRoutes(app: PreviewsV1App): void { const github = previewGitHub(c); const result = await cachedLookup( - c.env.CACHE_KV, + getPlatform(c).getCache("CACHE_KV"), `preview:pr:${number}`, () => - resolvePrPreview(github, number, c.env.CACHE_KV, (p) => - c.executionCtx.waitUntil(p) + resolvePrPreview( + github, + number, + getPlatform(c).getCache("CACHE_KV"), + getPlatform(c).waitUntil ), DEFAULT_CACHE_TTL_SECONDS, - (p) => c.executionCtx.waitUntil(p) + getPlatform(c).waitUntil ); return respondWithLookup(c, result, notFoundMessage(number), false); @@ -110,14 +115,17 @@ export function registerPrRoutes(app: PreviewsV1App): void { // GitHub API calls (PR->SHA, SHA->artifact, then the redirect) instead // of the 1 that's actually unavoidable. const artifact = await cachedLookup( - c.env.CACHE_KV, + getPlatform(c).getCache("CACHE_KV"), `preview:pr:${number}`, () => - resolvePrPreview(github, number, c.env.CACHE_KV, (p) => - c.executionCtx.waitUntil(p) + resolvePrPreview( + github, + number, + getPlatform(c).getCache("CACHE_KV"), + getPlatform(c).waitUntil ), DEFAULT_CACHE_TTL_SECONDS, - (p) => c.executionCtx.waitUntil(p) + getPlatform(c).waitUntil ); return respondWithDownloadRedirect( c, diff --git a/src/services/previews/v1/routes/respond.ts b/services/api/src/services/previews/v1/routes/respond.ts similarity index 100% rename from src/services/previews/v1/routes/respond.ts rename to services/api/src/services/previews/v1/routes/respond.ts diff --git a/src/services/previews/v1/schemas/previews.ts b/services/api/src/services/previews/v1/schemas/previews.ts similarity index 100% rename from src/services/previews/v1/schemas/previews.ts rename to services/api/src/services/previews/v1/schemas/previews.ts diff --git a/src/services/stats/v1/README.md b/services/api/src/services/stats/v1/README.md similarity index 100% rename from src/services/stats/v1/README.md rename to services/api/src/services/stats/v1/README.md diff --git a/src/services/stats/v1/dashboard.ts b/services/api/src/services/stats/v1/dashboard.ts similarity index 100% rename from src/services/stats/v1/dashboard.ts rename to services/api/src/services/stats/v1/dashboard.ts diff --git a/src/services/stats/v1/index.ts b/services/api/src/services/stats/v1/index.ts similarity index 98% rename from src/services/stats/v1/index.ts rename to services/api/src/services/stats/v1/index.ts index 266bf5e8..2becccfb 100644 --- a/src/services/stats/v1/index.ts +++ b/services/api/src/services/stats/v1/index.ts @@ -16,7 +16,7 @@ import { STATS_DASHBOARD_HTML } from "./dashboard"; import { getPlatform } from "../../../lib/middleware"; import { ICache } from "../../../lib/interfaces"; import { publicCacheKey, publicResponseCache } from "../../../lib/cache"; -import { logError, logInfo } from "../../../lib/logger"; +import { logError, logInfo } from "@fossbilling/logging"; import { GitHubError } from "../../../lib/github-errors"; type StatsEnv = { Bindings: CloudflareBindings }; @@ -225,7 +225,7 @@ registerCachedRoute("/data", async (c) => { platform.getCache("CACHE_KV"), platform.getEnv("GITHUB_TOKEN") || "", c.env.DOWNLOAD_BUCKET, - (p) => c.executionCtx.waitUntil(p) + platform.waitUntil ); if (result.error && result.stats.releaseSizes.length === 0) { diff --git a/src/services/stats/v1/interfaces.ts b/services/api/src/services/stats/v1/interfaces.ts similarity index 100% rename from src/services/stats/v1/interfaces.ts rename to services/api/src/services/stats/v1/interfaces.ts diff --git a/src/services/versions/v1/README.md b/services/api/src/services/versions/v1/README.md similarity index 100% rename from src/services/versions/v1/README.md rename to services/api/src/services/versions/v1/README.md diff --git a/src/services/versions/v1/index.ts b/services/api/src/services/versions/v1/index.ts similarity index 99% rename from src/services/versions/v1/index.ts rename to services/api/src/services/versions/v1/index.ts index 740721c4..e0d16abe 100644 --- a/src/services/versions/v1/index.ts +++ b/services/api/src/services/versions/v1/index.ts @@ -28,7 +28,7 @@ import { publicResponseCache, singleFlight } from "../../../lib/cache"; -import { logError, logWarn, logInfo } from "../../../lib/logger"; +import { logError, logWarn, logInfo } from "@fossbilling/logging"; import { GitHubError, AuthError, @@ -176,7 +176,7 @@ async function loadReleases( platform.getEnv("GITHUB_TOKEN") || "", c.env.DOWNLOAD_BUCKET, updateCache, - (p) => c.executionCtx.waitUntil(p) + platform.waitUntil ); } diff --git a/src/services/versions/v1/interfaces.ts b/services/api/src/services/versions/v1/interfaces.ts similarity index 100% rename from src/services/versions/v1/interfaces.ts rename to services/api/src/services/versions/v1/interfaces.ts diff --git a/src/services/versions/v1/r2.ts b/services/api/src/services/versions/v1/r2.ts similarity index 100% rename from src/services/versions/v1/r2.ts rename to services/api/src/services/versions/v1/r2.ts diff --git a/src/services/versions/v1/responses.ts b/services/api/src/services/versions/v1/responses.ts similarity index 100% rename from src/services/versions/v1/responses.ts rename to services/api/src/services/versions/v1/responses.ts diff --git a/services/api/test/deployment.node.test.ts b/services/api/test/deployment.node.test.ts new file mode 100644 index 00000000..68bf68f3 --- /dev/null +++ b/services/api/test/deployment.node.test.ts @@ -0,0 +1,127 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { + copyFileSync, + existsSync, + mkdtempSync, + readFileSync, + readdirSync, + rmSync, + symlinkSync +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath, URL } from "node:url"; +import { experimental_readRawConfig, type Unstable_RawConfig } from "wrangler"; +import { prepareDeployment } from "../scripts/prepare-deployment"; + +const service = fileURLToPath(new URL("../", import.meta.url)); +const resources = { + DB_EXTENSIONS: "11111111-1111-4111-8111-111111111111", + DB_CENTRAL_ALERTS: "22222222-2222-4222-8222-222222222222", + AUTH_KV: "a".repeat(32), + CACHE_KV: "b".repeat(32), + DOWNLOAD_BUCKET: "api-staging", + EXTENSIONS_FRONTEND: "api-staging" +}; +const folders: string[] = []; +afterEach(() => { + for (const folder of folders.splice(0)) + rmSync(folder, { recursive: true, force: true }); +}); +function fixture() { + const folder = mkdtempSync(join(tmpdir(), "api-deployment-")); + folders.push(folder); + copyFileSync(join(service, "wrangler.jsonc"), join(folder, "wrangler.jsonc")); + symlinkSync(join(service, "src"), join(folder, "src"), "dir"); + return folder; +} +const read = (folder: string, file: string) => + JSON.parse(readFileSync(join(folder, "dist", file), "utf8")); + +describe("API deployment artifact", () => { + it("keeps production and Preview targets separate and packages the exact SQL used by both", () => { + const folder = fixture(); + const source: Unstable_RawConfig = experimental_readRawConfig({ + config: join(folder, "wrangler.jsonc") + }).rawConfig; + prepareDeployment(folder, JSON.stringify(resources)); + const artifact = read(folder, "wrangler.json"); + expect(artifact.main).toBe("index.js"); + expect(artifact.no_bundle).toBe(true); + expect( + artifact.d1_databases.map((db: { database_id: string }) => db.database_id) + ).toEqual(source.d1_databases!.map((db) => db.database_id)); + expect( + artifact.previews.d1_databases.map( + (db: { binding: string; database_id: string }) => [ + db.binding, + db.database_id + ] + ) + ).toEqual( + source.d1_databases!.map((db) => [ + db.binding, + resources[db.binding as "DB_EXTENSIONS" | "DB_CENTRAL_ALERTS"] + ]) + ); + expect( + read(folder, "wrangler.preview-migrations.json").d1_databases + ).toEqual(artifact.previews.d1_databases); + expect(artifact.previews.kv_namespaces).toEqual([ + { binding: "AUTH_KV", id: resources.AUTH_KV }, + { binding: "CACHE_KV", id: resources.CACHE_KV } + ]); + expect(artifact.previews.r2_buckets).toEqual([ + { binding: "DOWNLOAD_BUCKET", bucket_name: resources.DOWNLOAD_BUCKET } + ]); + expect(artifact.previews.services).toEqual([ + { binding: "EXTENSIONS_FRONTEND", service: resources.EXTENSIONS_FRONTEND } + ]); + expect(artifact.previews.vars.EXTENSION_PACKAGES_MODE).toBe("disabled"); + for (const database of source.d1_databases!) { + const packaged = artifact.d1_databases.find( + (db: { binding: string }) => db.binding === database.binding + ); + const original = join(service, database.migrations_dir!); + const copied = join(folder, "dist", packaged.migrations_dir); + const sql = readdirSync(original) + .filter((name) => name.endsWith(".sql")) + .sort(); + expect(readdirSync(copied).sort()).toEqual(sql); + for (const name of sql) + expect(readFileSync(join(copied, name))).toEqual( + readFileSync(join(original, name)) + ); + } + // Rebuilding without Preview resources cannot reuse a stale migration target. + prepareDeployment(folder); + expect( + existsSync(join(folder, "dist/wrangler.preview-migrations.json")) + ).toBe(false); + expect(read(folder, "wrangler.json").previews.d1_databases).toBeUndefined(); + }); + it("rejects production resources and duplicated staging IDs before writing an artifact", () => { + const folder = fixture(); + const source: Unstable_RawConfig = experimental_readRawConfig({ + config: join(folder, "wrangler.jsonc") + }).rawConfig; + const overrides = [ + { DB_EXTENSIONS: source.d1_databases![0].database_id }, + { DB_EXTENSIONS: source.d1_databases![0].database_id!.toUpperCase() }, + { DB_EXTENSIONS: resources.DB_CENTRAL_ALERTS }, + { AUTH_KV: source.kv_namespaces![0].id }, + { CACHE_KV: source.kv_namespaces![1].id }, + { DOWNLOAD_BUCKET: source.r2_buckets![0].bucket_name }, + { EXTENSIONS_FRONTEND: source.services![0].service }, + { AUTH_KV: resources.CACHE_KV } + ]; + for (const override of overrides) { + expect(() => + prepareDeployment(folder, JSON.stringify({ ...resources, ...override })) + ).toThrow( + "API_PREVIEW_RESOURCES must use distinct non-production resources" + ); + expect(existsSync(join(folder, "dist/wrangler.json"))).toBe(false); + } + }); +}); diff --git a/test/env.d.ts b/services/api/test/env.d.ts similarity index 100% rename from test/env.d.ts rename to services/api/test/env.d.ts diff --git a/test/integration/app.test.ts b/services/api/test/integration/app.test.ts similarity index 100% rename from test/integration/app.test.ts rename to services/api/test/integration/app.test.ts diff --git a/test/integration/scheduled.test.ts b/services/api/test/integration/scheduled.test.ts similarity index 86% rename from test/integration/scheduled.test.ts rename to services/api/test/integration/scheduled.test.ts index 0a17d647..39b336ce 100644 --- a/test/integration/scheduled.test.ts +++ b/services/api/test/integration/scheduled.test.ts @@ -5,15 +5,15 @@ const mocks = vi.hoisted(() => ({ maintain: vi.fn(), report: vi.fn() })); -vi.mock("../../src/lib/adapters/cloudflare/package-jobs", () => ({ - consumePackageJobs: vi.fn(), - recoverPackageJobs: mocks.recover +vi.mock("../../src/lib/adapters/cloudflare/packaging-jobs", () => ({ + consumePackagingJobs: vi.fn(), + recoverPackagingJobs: mocks.recover })); vi.mock("../../src/services/extensions/v2/db/resource-maintenance", () => ({ maintainExtensionResources: mocks.maintain, reportExtensionResources: mocks.report })); -import app from "../../src/app"; +import app from "../../src/app/index"; afterEach(() => vi.restoreAllMocks()); describe("scheduled maintenance", () => { it("continues resource maintenance when package recovery fails", async () => { diff --git a/test/integration/versions/index.test.ts b/services/api/test/integration/versions/index.test.ts similarity index 99% rename from test/integration/versions/index.test.ts rename to services/api/test/integration/versions/index.test.ts index e705565f..4c79e760 100644 --- a/test/integration/versions/index.test.ts +++ b/services/api/test/integration/versions/index.test.ts @@ -5,7 +5,7 @@ import { waitOnExecutionContext } from "cloudflare:test"; import { env } from "cloudflare:workers"; -import app from "../../../src/app"; +import app from "../../../src/app/index"; const BYPASS_CACHE = { authorization: "test-bypass-cache" } as const; import { diff --git a/test/lib/adapters/cloudflare/cache.test.ts b/services/api/test/lib/adapters/cloudflare/cache.test.ts similarity index 100% rename from test/lib/adapters/cloudflare/cache.test.ts rename to services/api/test/lib/adapters/cloudflare/cache.test.ts diff --git a/test/lib/adapters/cloudflare/environment.test.ts b/services/api/test/lib/adapters/cloudflare/environment.test.ts similarity index 100% rename from test/lib/adapters/cloudflare/environment.test.ts rename to services/api/test/lib/adapters/cloudflare/environment.test.ts diff --git a/test/lib/adapters/node/cache.test.ts b/services/api/test/lib/adapters/node/cache.test.ts similarity index 100% rename from test/lib/adapters/node/cache.test.ts rename to services/api/test/lib/adapters/node/cache.test.ts diff --git a/test/lib/adapters/node/environment.test.ts b/services/api/test/lib/adapters/node/environment.test.ts similarity index 100% rename from test/lib/adapters/node/environment.test.ts rename to services/api/test/lib/adapters/node/environment.test.ts diff --git a/test/lib/adapters/node/index.test.ts b/services/api/test/lib/adapters/node/index.test.ts similarity index 100% rename from test/lib/adapters/node/index.test.ts rename to services/api/test/lib/adapters/node/index.test.ts diff --git a/test/lib/auth/assertion-helper.ts b/services/api/test/lib/auth/assertion-helper.ts similarity index 100% rename from test/lib/auth/assertion-helper.ts rename to services/api/test/lib/auth/assertion-helper.ts diff --git a/test/lib/auth/bearer-assertion.test.ts b/services/api/test/lib/auth/bearer-assertion.test.ts similarity index 96% rename from test/lib/auth/bearer-assertion.test.ts rename to services/api/test/lib/auth/bearer-assertion.test.ts index 44ef7c4d..fc70ee66 100644 --- a/test/lib/auth/bearer-assertion.test.ts +++ b/services/api/test/lib/auth/bearer-assertion.test.ts @@ -3,11 +3,11 @@ import { bearerAssertionVerifier, identitySyncAssertionVerifier } from "../../../src/lib/auth/bearer-assertion"; -import { logWarn } from "../../../src/lib/logger"; +import { logWarn } from "@fossbilling/logging"; import { PlatformContext } from "../../../src/lib/context"; import { base64UrlEncodeString, signAssertion } from "./assertion-helper"; -vi.mock("../../../src/lib/logger", () => ({ +vi.mock("@fossbilling/logging", () => ({ logError: vi.fn(), logWarn: vi.fn(), logInfo: vi.fn() @@ -18,20 +18,13 @@ const SECRET = "test-secret"; function platformWithSecret( secret: string | undefined, previousSecret?: string -): PlatformContext { +): Pick { return { - getQueue: () => { - throw new Error("No queue in auth test"); - }, - getCache: () => { - throw new Error("not implemented"); - }, getEnv: (key: string) => { if (key === "ASSERTION_SIGNING_SECRET") return secret; if (key === "ASSERTION_SIGNING_SECRET_PREVIOUS") return previousSecret; return undefined; - }, - raw: undefined as unknown as PlatformContext["raw"] + } }; } diff --git a/test/lib/auth/m2m-bearer.test.ts b/services/api/test/lib/auth/m2m-bearer.test.ts similarity index 97% rename from test/lib/auth/m2m-bearer.test.ts rename to services/api/test/lib/auth/m2m-bearer.test.ts index 762a023c..3c6071d7 100644 --- a/test/lib/auth/m2m-bearer.test.ts +++ b/services/api/test/lib/auth/m2m-bearer.test.ts @@ -16,11 +16,11 @@ const platform = { M2M_JWKS_URL: "https://auth.example.test/.well-known/jwks.json", M2M_AUDIENCE: "https://api.fossbilling.net" })[key] -} as unknown as PlatformContext; +} satisfies Pick; const emptyPlatform = { getEnv: () => undefined -} as unknown as PlatformContext; +} satisfies Pick; const verifier = m2mBearerVerifier; diff --git a/test/lib/auth/m2m-helper.ts b/services/api/test/lib/auth/m2m-helper.ts similarity index 100% rename from test/lib/auth/m2m-helper.ts rename to services/api/test/lib/auth/m2m-helper.ts diff --git a/test/lib/cache.test.ts b/services/api/test/lib/cache.test.ts similarity index 100% rename from test/lib/cache.test.ts rename to services/api/test/lib/cache.test.ts diff --git a/test/lib/context.test.ts b/services/api/test/lib/context.test.ts similarity index 73% rename from test/lib/context.test.ts rename to services/api/test/lib/context.test.ts index 86d305c9..0f17ee1d 100644 --- a/test/lib/context.test.ts +++ b/services/api/test/lib/context.test.ts @@ -1,5 +1,4 @@ import { describe, it, expect } from "vitest"; -import { Context } from "hono"; import { createPlatformContext } from "../../src/lib/context"; import { IPlatformBindings } from "../../src/lib/interfaces"; @@ -22,30 +21,22 @@ describe("createPlatformContext", () => { } }; - const mockHonoContext = { - get: () => ({}), - set: () => {}, - req: {}, - res: {} - } as unknown as Context; - - it("exposes the binding accessors and the raw context", () => { - const context = createPlatformContext(mockHonoContext, mockBindings); + it("exposes the binding accessors", () => { + const context = createPlatformContext(mockBindings, () => {}); expect(context.getCache("testCache")).toBe(mockBindings.caches.testCache); expect(context.getEnv("TEST_VAR")).toBe("test-value"); expect(context.getQueue("testQueue")).toBe(mockBindings.queues!.testQueue); - expect(context.raw).toBe(mockHonoContext); }); it("throws when a queue binding is missing", () => { - const context = createPlatformContext(mockHonoContext, mockBindings); + const context = createPlatformContext(mockBindings, () => {}); expect(() => context.getQueue("missing")).toThrow( "Queue binding 'missing' not found" ); }); it("throws when cache binding not found", () => { - const context = createPlatformContext(mockHonoContext, mockBindings); + const context = createPlatformContext(mockBindings, () => {}); expect(() => context.getCache("nonexistent")).toThrow( "Cache binding 'nonexistent' not found" diff --git a/test/lib/github-errors.test.ts b/services/api/test/lib/github-errors.test.ts similarity index 100% rename from test/lib/github-errors.test.ts rename to services/api/test/lib/github-errors.test.ts diff --git a/test/lib/middleware.test.ts b/services/api/test/lib/middleware.test.ts similarity index 100% rename from test/lib/middleware.test.ts rename to services/api/test/lib/middleware.test.ts diff --git a/test/lib/releases.test.ts b/services/api/test/lib/releases.test.ts similarity index 100% rename from test/lib/releases.test.ts rename to services/api/test/lib/releases.test.ts diff --git a/test/mocks/README.md b/services/api/test/mocks/README.md similarity index 100% rename from test/mocks/README.md rename to services/api/test/mocks/README.md diff --git a/test/mocks/github-releases.ts b/services/api/test/mocks/github-releases.ts similarity index 100% rename from test/mocks/github-releases.ts rename to services/api/test/mocks/github-releases.ts diff --git a/test/mocks/octokit.ts b/services/api/test/mocks/octokit.ts similarity index 100% rename from test/mocks/octokit.ts rename to services/api/test/mocks/octokit.ts diff --git a/test/services/central-alerts/v1/database.test.ts b/services/api/test/services/central-alerts/v1/database.test.ts similarity index 100% rename from test/services/central-alerts/v1/database.test.ts rename to services/api/test/services/central-alerts/v1/database.test.ts diff --git a/test/services/central-alerts/v1/index.test.ts b/services/api/test/services/central-alerts/v1/index.test.ts similarity index 99% rename from test/services/central-alerts/v1/index.test.ts rename to services/api/test/services/central-alerts/v1/index.test.ts index b4b17903..3ede01c8 100644 --- a/test/services/central-alerts/v1/index.test.ts +++ b/services/api/test/services/central-alerts/v1/index.test.ts @@ -4,7 +4,7 @@ import { waitOnExecutionContext } from "cloudflare:test"; import { env } from "cloudflare:workers"; -import app from "../../../../src/app"; +import app from "../../../../src/app/index"; import type { CentralAlertsResponse } from "../../../utils/test-types"; import { applyTestMigrations } from "../../../utils/apply-migrations"; diff --git a/test/services/extensions/v1/index.test.ts b/services/api/test/services/extensions/v1/index.test.ts similarity index 99% rename from test/services/extensions/v1/index.test.ts rename to services/api/test/services/extensions/v1/index.test.ts index 41f6adbc..92a8e618 100644 --- a/test/services/extensions/v1/index.test.ts +++ b/services/api/test/services/extensions/v1/index.test.ts @@ -6,7 +6,7 @@ import { waitOnExecutionContext } from "cloudflare:test"; import { env } from "cloudflare:workers"; -import app from "../../../../src/app"; +import app from "../../../../src/app/index"; import { getExtensionsDb } from "../../../../src/lib/db"; import { developers, diff --git a/test/services/extensions/v2/account.test.ts b/services/api/test/services/extensions/v2/account.test.ts similarity index 99% rename from test/services/extensions/v2/account.test.ts rename to services/api/test/services/extensions/v2/account.test.ts index 749359c8..ebb494f0 100644 --- a/test/services/extensions/v2/account.test.ts +++ b/services/api/test/services/extensions/v2/account.test.ts @@ -1,5 +1,5 @@ import { UUID_V7 } from "./db-fixtures"; -import app from "../../../../src/app"; +import app from "../../../../src/app/index"; import { env } from "cloudflare:workers"; import { createExecutionContext, diff --git a/test/services/extensions/v2/badges.test.ts b/services/api/test/services/extensions/v2/badges.test.ts similarity index 100% rename from test/services/extensions/v2/badges.test.ts rename to services/api/test/services/extensions/v2/badges.test.ts diff --git a/test/services/extensions/v2/claim-budget.test.ts b/services/api/test/services/extensions/v2/claim-budget.test.ts similarity index 100% rename from test/services/extensions/v2/claim-budget.test.ts rename to services/api/test/services/extensions/v2/claim-budget.test.ts diff --git a/test/services/extensions/v2/contract.test.ts b/services/api/test/services/extensions/v2/contract.test.ts similarity index 100% rename from test/services/extensions/v2/contract.test.ts rename to services/api/test/services/extensions/v2/contract.test.ts diff --git a/test/services/extensions/v2/db-fixtures.ts b/services/api/test/services/extensions/v2/db-fixtures.ts similarity index 100% rename from test/services/extensions/v2/db-fixtures.ts rename to services/api/test/services/extensions/v2/db-fixtures.ts diff --git a/test/services/extensions/v2/db-interceptor.ts b/services/api/test/services/extensions/v2/db-interceptor.ts similarity index 100% rename from test/services/extensions/v2/db-interceptor.ts rename to services/api/test/services/extensions/v2/db-interceptor.ts diff --git a/test/services/extensions/v2/developer-approval.test.ts b/services/api/test/services/extensions/v2/developer-approval.test.ts similarity index 100% rename from test/services/extensions/v2/developer-approval.test.ts rename to services/api/test/services/extensions/v2/developer-approval.test.ts diff --git a/test/services/extensions/v2/developer-profiles.test.ts b/services/api/test/services/extensions/v2/developer-profiles.test.ts similarity index 100% rename from test/services/extensions/v2/developer-profiles.test.ts rename to services/api/test/services/extensions/v2/developer-profiles.test.ts diff --git a/test/services/extensions/v2/directory-identity.test.ts b/services/api/test/services/extensions/v2/directory-identity.test.ts similarity index 100% rename from test/services/extensions/v2/directory-identity.test.ts rename to services/api/test/services/extensions/v2/directory-identity.test.ts diff --git a/test/services/extensions/v2/email.test.ts b/services/api/test/services/extensions/v2/email.test.ts similarity index 100% rename from test/services/extensions/v2/email.test.ts rename to services/api/test/services/extensions/v2/email.test.ts diff --git a/test/services/extensions/v2/extension-writes.test.ts b/services/api/test/services/extensions/v2/extension-writes.test.ts similarity index 100% rename from test/services/extensions/v2/extension-writes.test.ts rename to services/api/test/services/extensions/v2/extension-writes.test.ts diff --git a/test/services/extensions/v2/harness.ts b/services/api/test/services/extensions/v2/harness.ts similarity index 99% rename from test/services/extensions/v2/harness.ts rename to services/api/test/services/extensions/v2/harness.ts index c7e77ff6..99e85ca2 100644 --- a/test/services/extensions/v2/harness.ts +++ b/services/api/test/services/extensions/v2/harness.ts @@ -5,7 +5,7 @@ import { } from "cloudflare:test"; import { env } from "cloudflare:workers"; import { request as ghRequest } from "@octokit/request"; -import app from "../../../../src/app"; +import app from "../../../../src/app/index"; import { signAssertion } from "../../../lib/auth/assertion-helper"; import { MockGitHubRequest } from "../../../utils/test-types"; import { applyTestMigrations } from "../../../utils/apply-migrations"; diff --git a/test/services/extensions/v2/index.test.ts b/services/api/test/services/extensions/v2/index.test.ts similarity index 100% rename from test/services/extensions/v2/index.test.ts rename to services/api/test/services/extensions/v2/index.test.ts diff --git a/test/services/extensions/v2/manifest.test.ts b/services/api/test/services/extensions/v2/manifest.test.ts similarity index 100% rename from test/services/extensions/v2/manifest.test.ts rename to services/api/test/services/extensions/v2/manifest.test.ts diff --git a/test/services/extensions/v2/migrations.test.ts b/services/api/test/services/extensions/v2/migrations.test.ts similarity index 100% rename from test/services/extensions/v2/migrations.test.ts rename to services/api/test/services/extensions/v2/migrations.test.ts diff --git a/test/services/extensions/v2/moderation-correct.test.ts b/services/api/test/services/extensions/v2/moderation-correct.test.ts similarity index 100% rename from test/services/extensions/v2/moderation-correct.test.ts rename to services/api/test/services/extensions/v2/moderation-correct.test.ts diff --git a/test/services/extensions/v2/moderation-notify.test.ts b/services/api/test/services/extensions/v2/moderation-notify.test.ts similarity index 100% rename from test/services/extensions/v2/moderation-notify.test.ts rename to services/api/test/services/extensions/v2/moderation-notify.test.ts diff --git a/test/services/extensions/v2/moderation-revalidate.test.ts b/services/api/test/services/extensions/v2/moderation-revalidate.test.ts similarity index 95% rename from test/services/extensions/v2/moderation-revalidate.test.ts rename to services/api/test/services/extensions/v2/moderation-revalidate.test.ts index f2b3fb7f..05ae419a 100644 --- a/test/services/extensions/v2/moderation-revalidate.test.ts +++ b/services/api/test/services/extensions/v2/moderation-revalidate.test.ts @@ -46,9 +46,9 @@ function stubFrontend(status = 200): FetcherStub { return fetcher; } -function fetchCalls(fetcher: FetcherStub): Array<[unknown, RequestInit]> { +function fetchCalls(fetcher: FetcherStub): Array<[Request]> { return (fetcher.fetch as ReturnType).mock.calls as Array< - [unknown, RequestInit] + [Request] >; } @@ -136,16 +136,16 @@ describe("CDN cache revalidation on extension mutations", () => { expect(res.status).toBe(200); expect(fetchCalls(fetcher)).toHaveLength(1); - const [url, init] = fetchCalls(fetcher)[0]; - expect(String(url)).toBe( + const [request] = fetchCalls(fetcher)[0]; + expect(request.url).toBe( "https://extensions.fossbilling.org/api/revalidate" ); - expect(init.method).toBe("POST"); - expect(init.headers).toMatchObject({ + expect(request.method).toBe("POST"); + expect(Object.fromEntries(request.headers)).toMatchObject({ authorization: `Bearer ${SECRET}`, "content-type": "application/json" }); - expect(JSON.parse(String(init.body))).toEqual({ + expect(await request.json()).toEqual({ tags: ["extensions", "developers"] }); }); @@ -163,12 +163,12 @@ describe("CDN cache revalidation on extension mutations", () => { expect(res.status).toBe(200); expect(fetchCalls(fetcher)).toHaveLength(1); - const [url, init] = fetchCalls(fetcher)[0]; - expect(String(url)).toBe( + const [request] = fetchCalls(fetcher)[0]; + expect(request.url).toBe( "https://extensions.fossbilling.org/api/revalidate" ); - expect(init.method).toBe("POST"); - expect(JSON.parse(String(init.body))).toEqual({ + expect(request.method).toBe("POST"); + expect(await request.json()).toEqual({ tags: ["extensions", "developers"] }); }); @@ -338,10 +338,10 @@ describe("CDN cache revalidation on extension mutations", () => { }); }); - it("still delists when the purge request throws", async () => { + it("still delists when the frontend binding throws synchronously", async () => { await seedModAndExtension(); const fetcher: FetcherStub = { - fetch: vi.fn(async () => { + fetch: vi.fn(() => { throw new Error("binding unavailable"); }) }; diff --git a/test/services/extensions/v2/moderation-revocation.test.ts b/services/api/test/services/extensions/v2/moderation-revocation.test.ts similarity index 100% rename from test/services/extensions/v2/moderation-revocation.test.ts rename to services/api/test/services/extensions/v2/moderation-revocation.test.ts diff --git a/test/services/extensions/v2/moderation.test.ts b/services/api/test/services/extensions/v2/moderation.test.ts similarity index 100% rename from test/services/extensions/v2/moderation.test.ts rename to services/api/test/services/extensions/v2/moderation.test.ts diff --git a/test/services/extensions/v2/ownership.test.ts b/services/api/test/services/extensions/v2/ownership.test.ts similarity index 100% rename from test/services/extensions/v2/ownership.test.ts rename to services/api/test/services/extensions/v2/ownership.test.ts diff --git a/test/services/extensions/v2/package-build.node.test.ts b/services/api/test/services/extensions/v2/package-build.node.test.ts similarity index 90% rename from test/services/extensions/v2/package-build.node.test.ts rename to services/api/test/services/extensions/v2/package-build.node.test.ts index 97806995..06746310 100644 --- a/test/services/extensions/v2/package-build.node.test.ts +++ b/services/api/test/services/extensions/v2/package-build.node.test.ts @@ -7,6 +7,8 @@ import { ZipReader, ZipWriter } from "@zip.js/zip.js"; +import { type ProcessingInput } from "@fossbilling/extension-packaging/processor"; +import { processPackage } from "@fossbilling/extension-packaging/process"; import { prepareQueuedPackage } from "../../../../src/services/extensions/v2/packages/prepare"; import type { PackageRepository, @@ -14,9 +16,9 @@ import type { SourceRow, IconObject } from "../../../../src/services/extensions/v2/packages/repository"; -import type { PackageJob } from "../../../../src/services/extensions/v2/packages/jobs"; -import type { PackageStore } from "../../../../src/services/extensions/v2/packages/store"; -import { sha256 } from "../../../../src/services/extensions/v2/package-builder"; +import type { PackagingJob } from "../../../../src/services/extensions/v2/packages/jobs"; +import type { PackageStore } from "@fossbilling/extension-packaging/store"; +import { sha256 } from "@fossbilling/extension-packaging/checksum"; const zipOptions = { useWebWorkers: false, useCompressionStream: true }; describe("package preparation outside Cloudflare", () => { @@ -51,7 +53,8 @@ describe("package preparation outside Cloudflare", () => { findArtifact: async (_sourceId, hash) => artifacts.get(hash) ?? null, hasIcon: async () => icons !== null, reserveObject: async () => {}, - removeObject: unexpected, + setObjectSize: async () => {}, + removeObject: async () => {}, claimCleanup: unexpected, finishCleanup: unexpected, linkSource: async (_owner, _manifest, input) => (source ??= input), @@ -116,7 +119,7 @@ describe("package preparation outside Cloudflare", () => { const upload = await writer.close(); const uploadId = crypto.randomUUID(); objects.set(uploadId, upload); - const job: PackageJob = { + const job: PackagingJob = { id: crypto.randomUUID(), fingerprint: "node", owner, @@ -134,7 +137,10 @@ describe("package preparation outside Cloudflare", () => { const services = { repository, store, - images, + processor: { + process: (input: ProcessingInput) => + processPackage(input, store, images) + }, baseUrl: "https://directory.example.test/extensions/v2" }; const id = await prepareQueuedPackage(services, job); diff --git a/test/services/extensions/v2/packages.test.ts b/services/api/test/services/extensions/v2/packages.test.ts similarity index 86% rename from test/services/extensions/v2/packages.test.ts rename to services/api/test/services/extensions/v2/packages.test.ts index 8db2efeb..875f4c6e 100644 --- a/test/services/extensions/v2/packages.test.ts +++ b/services/api/test/services/extensions/v2/packages.test.ts @@ -1,9 +1,12 @@ -import { PackageJobsDatabase } from "../../../../src/services/extensions/v2/db/package-jobs"; -import { processPackageJob } from "../../../../src/services/extensions/v2/packages/jobs"; +import { processPackage } from "@fossbilling/extension-packaging/process"; +import { ProcessingInputSchema } from "@fossbilling/extension-packaging/processor"; +import { PackageError } from "@fossbilling/extension-packaging/errors"; +import { PackagingJobsDatabase } from "../../../../src/services/extensions/v2/db/packaging-jobs"; +import { processPackagingJob } from "../../../../src/services/extensions/v2/packages/jobs"; import { - recoverPackageJobs, + recoverPackagingJobs, packagePreparation -} from "../../../../src/lib/adapters/cloudflare/package-jobs"; +} from "../../../../src/lib/adapters/cloudflare/packaging-jobs"; import { prepareQueuedPackage } from "../../../../src/services/extensions/v2/packages/prepare"; import { beforeEach, afterEach, describe, expect, it, vi } from "vitest"; import { env } from "cloudflare:workers"; @@ -21,7 +24,7 @@ import { ZipReader, TextWriter } from "@zip.js/zip.js"; -import app from "../../../../src/app"; +import app from "../../../../src/app/index"; import { signAccessToken, activateM2mJwks, @@ -31,7 +34,7 @@ import { PackageArtifactSchema } from "../../../../src/services/extensions/v2/sc import { ManifestSchema } from "../../../../src/services/extensions/v2/schemas/manifest"; import { ExtensionContentSchema } from "../../../../src/services/extensions/v2/schemas/extensions"; import { contentManifest } from "../../../../src/services/extensions/v2/packages/manifest"; -import { sha256 } from "../../../../src/services/extensions/v2/package-builder"; +import { sha256 } from "@fossbilling/extension-packaging/checksum"; import { setupExtensionsV2Tests, authHeaders, @@ -57,9 +60,25 @@ const manifest = () => ); const packageKey = (id: string) => `/test-packages/local/packages/${id}`; let b2: ReturnType; +const preparedSvg = + ''; +const png = Uint8Array.from( + atob( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAACXBIWXMAAAPoAAAD6AG1e1JrAAAADUlEQVQImWNQTvnfAAAEOwIHOVGdNAAAAABJRU5ErkJggg==" + ), + (c) => c.charCodeAt(0) +); +const avif = Uint8Array.from( + atob( + "AAAAHGZ0eXBhdmlmAAAAAG1pZjFhdmlmbWlhZgAAAXBtZXRhAAAAAAAAACFoZGxyAAAAAAAAAABwaWN0AAAAAAAAAAAAAAAAAAAAADRpbG9jAAAAAERAAAIAAQAAAAABlAABAAAAAAAAACMAAgAAAAABtwABAAAAAAAAABAAAAA4aWluZgAAAAAAAgAAABVpbmZlAgAAAAABAABhdjAxAAAAABVpbmZlAgAAAAACAABhdjAxAAAAAA5waXRtAAAAAAABAAAAr2lwcnAAAACKaXBjbwAAAAxhdjFDgSACAAAAABRpc3BlAAAAAAAAAAEAAAABAAAAEHBpeGkAAAAAAwgICAAAAAxhdjFDgQAcAAAAAA5waXhpAAAAAAEIAAAAOGF1eEMAAAAAdXJuOm1wZWc6bXBlZ0I6Y2ljcDpzeXN0ZW1zOmF1eGlsaWFyeTphbHBoYQAAAAAdaXBtYQAAAAAAAAACAAEDgQIDAAIEhAIFhgAAABppcmVmAAAAAAAAAA5hdXhsAAIAAQABAAAAO21kYXQSAAoHOAAGkBDQaTIWGUJjBMAANAAAkEDJHGFRR0szJUz6VhIACgQYAAYVMgYYAAABQA8=" + ), + (c) => c.charCodeAt(0) +); function mockB2() { Object.assign(env, { + EXTENSION_PACKAGER_URL: "https://processor.example.test/process", + EXTENSION_PACKAGER_SECRET: "test-processor-secret", EXTENSION_PACKAGES_S3_ENDPOINT: "https://s3.example.test", EXTENSION_PACKAGES_S3_REGION: "test-region", EXTENSION_PACKAGES_S3_BUCKET: "test-packages", @@ -73,6 +92,23 @@ function mockB2() { EXTENSION_ASSETS_S3_SECRET_ACCESS_KEY: "test-secret", EXTENSION_ASSETS_PUBLIC_URL: "https://extensions-assets.fossbilling.net" }); + // The remote runner's conversion is tested with Sharp in the Node suite. + // Here it supplies prepared fixtures so these tests cover API publication. + const images = { + prepare: vi.fn(async (_icon: Uint8Array, isSvg: boolean) => + isSvg + ? [ + { + format: "svg" as const, + bytes: new Uint8Array(new TextEncoder().encode(preparedSvg)) + } + ] + : [ + { format: "avif" as const, bytes: avif }, + { format: "png" as const, bytes: png } + ] + ) + }; const calls: Request[] = []; const objects = new Map< string, @@ -84,6 +120,28 @@ function mockB2() { >(); const fetch = vi.fn(async (request: Request) => { const url = new URL(request.url); + if (url.hostname === "processor.example.test") { + const input = ProcessingInputSchema.parse(await request.json()); + const { b2ObjectStore } = + await import("@fossbilling/extension-packaging/adapters/b2"); + const store = b2ObjectStore((name) => Reflect.get(env, name), "packages"); + try { + return Response.json({ + result: await processPackage(input, store, images) + }); + } catch (error) { + const e = error as { code?: string; message: string; status?: number }; + return Response.json( + { + error: { + code: e.code, + message: e.message + } + }, + { status: e.status ?? 503 } + ); + } + } calls.push(request.clone()); const key = url.pathname; if (request.method === "PUT") { @@ -112,7 +170,7 @@ function mockB2() { }); }); vi.stubGlobal("fetch", fetch); - return { calls, objects, fetch }; + return { calls, objects, fetch, images }; } beforeEach(async () => { @@ -150,10 +208,20 @@ afterEach(() => { "PREFIX" ]) Reflect.deleteProperty(env, `EXTENSION_PACKAGES_S3_${suffix}`); + Reflect.deleteProperty(env, "EXTENSION_PACKAGER_URL"); + Reflect.deleteProperty(env, "EXTENSION_PACKAGER_SECRET"); env.EXTENSION_PACKAGES_MODE = "disabled"; // Assert outside the adapter's catch, including calls with injected failures. for (const request of requests) { const url = new URL(request.url); + if (url.hostname === "processor.example.test") { + expect(request.method).toBe("POST"); + expect(request.headers.get("Authorization")).toBe( + "Bearer test-processor-secret" + ); + expect(request.cache).toBe("no-store"); + continue; + } expect(url.hostname).toBe("s3.example.test"); expect(request.headers.get("Authorization")).toMatch(/^AWS4-HMAC-SHA256 /); expect(["PUT", "GET", "HEAD", "DELETE"]).toContain(request.method); @@ -198,14 +266,14 @@ async function upload( async function finishJob(response: Response): Promise { if (response.status !== 202) return response; const { result } = (await response.json()) as { result: { id: string } }; - const jobs = new PackageJobsDatabase(env.DB_EXTENSIONS); + const jobs = new PackagingJobsDatabase(env.DB_EXTENSIONS); let job = (await jobs.get(result.id))!; for ( let attempt = 0; attempt < 100 && job.status !== "ready" && job.status !== "failed"; attempt++ ) { - await processPackageJob(jobs, result.id, (job) => + await processPackagingJob(jobs, result.id, (job) => prepareQueuedPackage(packagePreparation(env), job) ); job = (await jobs.get(result.id))!; @@ -291,7 +359,9 @@ describe("publisher package builds", () => { vi.stubGlobal("fetch", (input: RequestInfo | URL, init?: RequestInit) => { const request = input instanceof Request ? input : new Request(input, init); - return new URL(request.url).hostname === "s3.example.test" + return ["s3.example.test", "processor.example.test"].includes( + new URL(request.url).hostname + ) ? b2.fetch(request) : jwksFetch(input, init); }); @@ -333,7 +403,7 @@ describe("publisher package builds", () => { const headers = { Authorization: `Bearer ${await signAccessToken({ clientId: "package-publisher", scopes: ["extensions:push"] })}` }; - const pending = await get(`${root}/package-jobs/${job.id}`, headers); + const pending = await get(`${root}/packaging-jobs/${job.id}`, headers); expect(pending.status).toBe(200); expect(await pending.json()).toMatchObject({ result: { status: "queued", artifact: null } @@ -341,7 +411,7 @@ describe("publisher package builds", () => { expect( (await finishJob(Response.json({ result: job }, { status: 202 }))).status ).toBe(200); - const ready = await get(`${root}/package-jobs/${job.id}`, headers); + const ready = await get(`${root}/packaging-jobs/${job.id}`, headers); expect(ready.status).toBe(200); expect(await ready.json()).toMatchObject({ result: { @@ -354,9 +424,9 @@ describe("publisher package builds", () => { "UPDATE m2m_publisher_clients SET revoked_at=CURRENT_TIMESTAMP WHERE client_id='package-publisher'" ) .run(); - expect((await get(`${root}/package-jobs/${job.id}`, headers)).status).toBe( - 403 - ); + expect( + (await get(`${root}/packaging-jobs/${job.id}`, headers)).status + ).toBe(403); }); it("builds under the mapped developer while retaining owner privacy", async () => { const response = await machineBuild(); @@ -482,11 +552,13 @@ describe("queued package preparation", () => { expect( ((await repeat.json()) as { result: { id: string } }).result.id ).toBe(job.id); - expect((await get(`${root}/package-jobs/${job.id}`, {})).status).toBe(401); + expect((await get(`${root}/packaging-jobs/${job.id}`, {})).status).toBe( + 401 + ); expect( ( await get( - `${root}/package-jobs/${job.id}`, + `${root}/packaging-jobs/${job.id}`, await authHeaders("stranger") ) ).status @@ -506,7 +578,7 @@ describe("queued package preparation", () => { retryMessages: [] }); const status = await get( - `${root}/package-jobs/${job.id}`, + `${root}/packaging-jobs/${job.id}`, await authHeaders("owner") ); expect( @@ -545,13 +617,13 @@ describe("queued package preparation", () => { expect( (await upload(manifest(), undefined, undefined, "owner", false)).status ).toBe(503); - const jobs = new PackageJobsDatabase(db); + const jobs = new PackagingJobsDatabase(db); const ids = await jobs.pending(); expect(ids).toHaveLength(1); - const { dispatchPackageJob } = + const { dispatchPackagingJob } = await import("../../../../src/services/extensions/v2/packages/jobs"); const send = vi.fn().mockResolvedValue(undefined); - await dispatchPackageJob(jobs, { send }, ids[0]); + await dispatchPackagingJob(jobs, { send }, ids[0]); expect(send).toHaveBeenCalledWith({ jobId: ids[0] }); expect(await jobs.pending()).toEqual([]); }); @@ -569,9 +641,9 @@ describe("queued package preparation", () => { vi.mocked(env.EXTENSION_PACKAGE_QUEUE.send) .mockClear() .mockRejectedValueOnce(new Error("queue unavailable")); - await recoverPackageJobs(env); + await recoverPackagingJobs(env); expect(env.EXTENSION_PACKAGE_QUEUE.send).toHaveBeenCalledTimes(2); - expect(await new PackageJobsDatabase(db).pending()).toHaveLength(1); + expect(await new PackagingJobsDatabase(db).pending()).toHaveLength(1); expect(logged).toHaveBeenCalledWith( expect.objectContaining({ context: expect.objectContaining({ error: "queue unavailable" }) @@ -588,13 +660,13 @@ describe("queued package preparation", () => { false ); const { result } = (await response.json()) as { result: { id: string } }; - const jobs = new PackageJobsDatabase(db); + const jobs = new PackagingJobsDatabase(db); const job = (await jobs.get(result.id))!; const key = packageKey(job.uploadObjectId!); const staged = b2.objects.get(key)!; b2.objects.delete(key); expect( - await processPackageJob(jobs, job.id, (job) => + await processPackagingJob(jobs, job.id, (job) => prepareQueuedPackage(packagePreparation(env), job) ) ).toBe("retry"); @@ -605,7 +677,7 @@ describe("queued package preparation", () => { }); b2.objects.set(key, staged); expect( - await processPackageJob(jobs, job.id, (job) => + await processPackagingJob(jobs, job.id, (job) => prepareQueuedPackage(packagePreparation(env), job) ) ).toBe("ack"); @@ -626,7 +698,7 @@ describe("queued package preparation", () => { const { result: job } = (await response.json()) as { result: { id: string }; }; - const jobs = new PackageJobsDatabase(db); + const jobs = new PackagingJobsDatabase(db); expect(await jobs.claim(job.id, "first")).not.toBeNull(); expect(await jobs.claim(job.id, "second")).toBeNull(); await db @@ -662,15 +734,15 @@ describe("queued package preparation", () => { const { result: job } = (await response.json()) as { result: { id: string }; }; - const jobs = new PackageJobsDatabase(db); + const jobs = new PackagingJobsDatabase(db); const prepare = vi.fn( ( - job: import("../../../../src/services/extensions/v2/packages/jobs").PackageJob + job: import("../../../../src/services/extensions/v2/packages/jobs").PackagingJob ) => prepareQueuedPackage(packagePreparation(env), job) ); await Promise.all([ - processPackageJob(jobs, job.id, prepare), - processPackageJob(jobs, job.id, prepare) + processPackagingJob(jobs, job.id, prepare), + processPackagingJob(jobs, job.id, prepare) ]); expect(prepare).toHaveBeenCalledTimes(1); expect((await jobs.get(job.id))!.status).toBe("ready"); @@ -696,14 +768,14 @@ describe("stored directory packages", () => { ); expect(revised.source_id).toBe(first.source_id); const uploads = requests.filter((request) => request.method === "PUT"); - expect(uploads).toHaveLength(4); // source, two manifests and one shared icon + expect(uploads).toHaveLength(5); // source, two manifests, staged icon and shared public icon expect( uploads.filter((request) => new URL(request.url).pathname.startsWith( "/test-packages/local/packages/" ) ) - ).toHaveLength(3); + ).toHaveLength(4); expect( uploads.filter((request) => new URL(request.url).pathname.startsWith("/test-assets/extensions/") @@ -789,8 +861,11 @@ describe("stored directory packages", () => { const preview = await get(path, await authHeaders("owner")); expect(preview.status).toBe(200); expect(preview.headers.get("Content-Type")).toBe("image/svg+xml"); - expect(preview.headers.get("Content-Security-Policy")).toContain("sandbox"); - expect(await preview.text()).toContain(" { .run(); expect((await get(path, {})).status).toBe(404); }); - it("sanitizes SVG scripts, events, foreign content, and stylesheets while preserving vector artwork", async () => { - const original = ` - -
HTML
- - - -
`; - const writer = new ZipWriter(new Uint8ArrayWriter(), options); - await writer.add("icon.svg", new TextReader(original)); - const artifact = await built(manifest(), await writer.close()); - const response = await get( - `${root}/packages/${artifact.id}/icon`, - await authHeaders("owner") - ); - const svg = await response.text(); - expect(response.headers.get("Content-Type")).toBe("image/svg+xml"); - expect(response.headers.get("Content-Security-Policy")).toBe( - "sandbox; default-src 'none'; style-src 'unsafe-inline'; img-src data:" - ); - expect(response.headers.get("X-Content-Type-Options")).toBe("nosniff"); - expect(svg).not.toMatch( - /script|onload|foreignObject|untrusted\.example|xml-stylesheet/ - ); - expect(svg).toContain("linearGradient"); - expect(svg).toContain("url(#paint)"); - expect(svg).toContain('viewBox="0 0 24 24"'); - const row = await db - .prepare( - "SELECT svg_object_id,avif_object_id,png_object_id FROM package_icons WHERE source_id=?" - ) - .bind(artifact.source_id) - .first<{ - svg_object_id: string; - avif_object_id: null; - png_object_id: null; - }>(); - expect(row?.svg_object_id).toMatch( - /^extensions\/[a-f0-9-]{36}\/icons\/[a-f0-9-]{36}\.svg$/ - ); - expect(row?.avif_object_id).toBeNull(); - expect(row?.png_object_id).toBeNull(); - const download = await get( - `${root}/packages/${artifact.id}/download`, - await authHeaders("owner") - ); - const zip = new ZipReader( - new Uint8ArrayReader(new Uint8Array(await download.arrayBuffer())), - options - ); - try { - const icon = (await zip.getEntries()).find( - (entry) => entry.filename === "icon.svg" - )!; - if (icon.directory) throw new Error("Expected icon file"); - expect(await icon.getData(new TextWriter())).toBe(original); - } finally { - await zip.close(); - } - }); - it.each([ - "', - '" - ])("rejects malformed, non-SVG, and oversized SVG files %#", async (svg) => { - const writer = new ZipWriter(new Uint8ArrayWriter(), options); - await writer.add("icon.svg", new TextReader(svg)); - expect((await upload(manifest(), await writer.close())).status).toBe(422); - expect( - await db.prepare("SELECT count(*) AS count FROM package_sources").first() - ).toEqual({ count: 0 }); - }); - it("sanitizes SVG content even with a raster filename", async () => { - const writer = new ZipWriter(new Uint8ArrayWriter(), options); - await writer.add( - "icon.png", - new TextReader( - '' - ) - ); - const artifact = await built(manifest(), await writer.close()); - const response = await get( - `${root}/packages/${artifact.id}/icon`, - await authHeaders("owner") - ); - expect(response.headers.get("Content-Type")).toBe("image/svg+xml"); - expect(await response.text()).not.toContain("onload"); + it("serves prepared AVIF or PNG according to the visitor's Accept header", async () => { + const artifact = await built(manifest(), await iconSource("png")); + const path = `${root}/packages/${artifact.id}/icon`; + const headers = await authHeaders("owner"); + const response = await get(path, headers); + expect(response.headers.get("Content-Type")).toBe("image/png"); + expect(new Uint8Array(await response.arrayBuffer())).toEqual(png); + const preferred = await get(path, { + ...headers, + Accept: "image/avif,image/png" + }); + expect(preferred.headers.get("Content-Type")).toBe("image/avif"); + expect(new Uint8Array(await preferred.arrayBuffer())).toEqual(avif); + const excluded = await get(path, { + ...headers, + Accept: "image/avif;q=0,image/png" + }); + expect(excluded.headers.get("Content-Type")).toBe("image/png"); }); - it.each(["png", "webp", "avif"] as const)( - "converts icon.%s to bounded AVIF and PNG images", - async (format) => { - const svg = - ''; - const image = await env - .PACKAGE_IMAGES!.input(new Blob([svg]).stream()) - .output({ format: `image/${format}` }); - const original = new Uint8Array(await image.response().arrayBuffer()); - const writer = new ZipWriter(new Uint8ArrayWriter(), options); - await writer.add(`icon.${format}`, new Uint8ArrayReader(original)); - const artifact = await built(manifest(), await writer.close()); - const response = await get( - `${root}/packages/${artifact.id}/icon`, - await authHeaders("owner") - ); - const avif = await get(`${root}/packages/${artifact.id}/icon`, { - ...(await authHeaders("owner")), - Accept: "image/avif,image/png" - }); - expect(avif.headers.get("Content-Type")).toBe("image/avif"); - expect( - new TextDecoder().decode((await avif.arrayBuffer()).slice(4, 12)) - ).toBe("ftypavif"); - expect( - ( - await get(`${root}/packages/${artifact.id}/icon`, { - ...(await authHeaders("owner")), - Accept: "image/avif;q=0,image/png" - }) - ).headers.get("Content-Type") - ).toBe("image/png"); - const info = await env.PACKAGE_IMAGES!.info(response.body!); - expect(info).toMatchObject({ - format: "image/png", - width: 256, - height: 128 - }); - const download = await get( - `${root}/packages/${artifact.id}/download`, - await authHeaders("owner") - ); - const zip = new ZipReader( - new Uint8ArrayReader(new Uint8Array(await download.arrayBuffer())), - options - ); - try { - const entry = (await zip.getEntries()).find( - (entry) => entry.filename === `icon.${format}` - )!; - if (entry.directory) throw new Error("Expected icon file"); - expect(await entry.getData(new Uint8ArrayWriter())).toEqual(original); - } finally { - await zip.close(); - } - } - ); it("lets moderators rebuild published source for a correction without uploading new files", async () => { const artifact = await built(); @@ -1017,7 +964,14 @@ describe("stored directory packages", () => { ).toBe(200); }); - it("rejects invalid icons without reserving the source version", async () => { + it("propagates runner icon rejection without reserving the source version", async () => { + b2.images.prepare.mockRejectedValueOnce( + new PackageError( + "INVALID_ICON", + "Package icon could not be processed", + 422 + ) + ); const writer = new ZipWriter(new Uint8ArrayWriter(), options); await writer.add("icon.png", new TextReader("not an image")); const response = await upload(manifest(), await writer.close()); @@ -1034,18 +988,7 @@ describe("stored directory packages", () => { async function iconSource(format: "svg" | "png" = "svg") { const svg = ''; - const bytes = - format === "svg" - ? new TextEncoder().encode(svg) - : new Uint8Array( - await ( - await env - .PACKAGE_IMAGES!.input(new Blob([svg]).stream()) - .output({ format: "image/png" }) - ) - .response() - .arrayBuffer() - ); + const bytes = format === "svg" ? new TextEncoder().encode(svg) : png; const writer = new ZipWriter(new Uint8ArrayWriter(), options); await writer.add( `icon.${format}`, @@ -1624,7 +1567,7 @@ describe("stored directory packages", () => { ); expect(response.status).toBe(202); expect(response.headers.get("Location")).toMatch( - /^https:\/\/api\.fossbilling\.net\/extensions\/v2\/package-jobs\// + /^https:\/\/api\.fossbilling\.net\/extensions\/v2\/packaging-jobs\// ); expect(requests.length).toBeGreaterThan(0); expect( @@ -1636,6 +1579,40 @@ describe("stored directory packages", () => { Object.assign(env, { ENVIRONMENT: "local" }); }); + it("rejects missing or malformed processor configuration before accepting an upload", async () => { + for (const overrides of [ + { EXTENSION_PACKAGER_URL: "" }, + { EXTENSION_PACKAGER_URL: "not a URL" }, + { + EXTENSION_PACKAGER_URL: "https://processor.example.test/process", + EXTENSION_PACKAGER_SECRET: "" + } + ]) { + Object.assign(env, overrides); + const response = await upload(); + expect(response.status).toBe(503); + expect(await response.json()).toMatchObject({ + error: { code: "PACKAGES_DISABLED" } + }); + } + expect( + await db.prepare("SELECT count(*) AS count FROM package_jobs").first() + ).toEqual({ count: 0 }); + expect( + await db.prepare("SELECT count(*) AS count FROM package_objects").first() + ).toEqual({ count: 0 }); + }); + + it("admits a small package with 30 MiB of remaining storage headroom", async () => { + await db + .prepare( + "INSERT INTO package_objects (id,created_by,bytes) VALUES (?,'owner',?)" + ) + .bind(crypto.randomUUID(), 482 * 1024 * 1024) + .run(); + expect((await upload()).status).toBe(200); + }); + it("checks account ownership, upload bounds, invalid archives and disabled storage", async () => { expect((await upload(manifest(), new Uint8Array([1, 2, 3]))).status).toBe( 422 diff --git a/test/services/extensions/v2/package-jobs.node.test.ts b/services/api/test/services/extensions/v2/packaging-jobs.node.test.ts similarity index 80% rename from test/services/extensions/v2/package-jobs.node.test.ts rename to services/api/test/services/extensions/v2/packaging-jobs.node.test.ts index 818706a3..b53055fd 100644 --- a/test/services/extensions/v2/package-jobs.node.test.ts +++ b/services/api/test/services/extensions/v2/packaging-jobs.node.test.ts @@ -1,14 +1,14 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { - dispatchPackageJob, - processPackageJob, - type PackageJob, - type PackageJobRepository + dispatchPackagingJob, + processPackagingJob, + type PackagingJob, + type PackagingJobRepository } from "../../../../src/services/extensions/v2/packages/jobs"; -import { PackageError } from "../../../../src/services/extensions/v2/packages/errors"; +import { PackageError } from "@fossbilling/extension-packaging/errors"; function repository() { - const job: PackageJob = { + const job: PackagingJob = { id: crypto.randomUUID(), fingerprint: "test", owner: { @@ -39,7 +39,7 @@ function repository() { updatedAt: "now" }; let activeLease: string | null = null; - const jobs: PackageJobRepository = { + const jobs: PackagingJobRepository = { get: async () => ({ ...job }), claim: async (_id, lease) => { if (job.status !== "queued") return null; @@ -70,11 +70,11 @@ function repository() { afterEach(() => vi.restoreAllMocks()); -describe("provider-independent package job processing", () => { +describe("provider-independent packaging job processing", () => { it("publishes only an identifier through the queue interface", async () => { const { jobs, job } = repository(); const send = vi.fn().mockResolvedValue(undefined); - await dispatchPackageJob(jobs, { send }, job.id); + await dispatchPackagingJob(jobs, { send }, job.id); expect(send).toHaveBeenCalledWith({ jobId: job.id }); expect(jobs.markEnqueued).toHaveBeenCalledWith(job.id); }); @@ -82,7 +82,7 @@ describe("provider-independent package job processing", () => { const { jobs, job } = repository(); job.status = "processing"; const prepare = vi.fn(); - expect(await processPackageJob(jobs, job.id, prepare)).toBe("busy"); + expect(await processPackagingJob(jobs, job.id, prepare)).toBe("busy"); expect(job.status).toBe("processing"); expect(job.attempts).toBe(0); expect(prepare).not.toHaveBeenCalled(); @@ -91,7 +91,7 @@ describe("provider-independent package job processing", () => { const { jobs, job } = repository(); job.attempts = 3; const prepare = vi.fn(); - expect(await processPackageJob(jobs, job.id, prepare)).toBe("ack"); + expect(await processPackagingJob(jobs, job.id, prepare)).toBe("ack"); expect(job.attempts).toBe(4); expect(job.status).toBe("failed"); expect(job.error).toMatchObject({ @@ -103,8 +103,8 @@ describe("provider-independent package job processing", () => { it("does not run completed jobs twice", async () => { const { jobs, job } = repository(); const prepare = vi.fn().mockResolvedValue("artifact"); - expect(await processPackageJob(jobs, job.id, prepare)).toBe("ack"); - expect(await processPackageJob(jobs, job.id, prepare)).toBe("ack"); + expect(await processPackagingJob(jobs, job.id, prepare)).toBe("ack"); + expect(await processPackagingJob(jobs, job.id, prepare)).toBe("ack"); expect(prepare).toHaveBeenCalledTimes(1); expect(job.artifactId).toBe("artifact"); }); @@ -112,9 +112,9 @@ describe("provider-independent package job processing", () => { const { jobs, job } = repository(); const logged = vi.spyOn(console, "error").mockImplementation(() => {}); const prepare = vi.fn().mockRejectedValue(new Error("storage unavailable")); - expect(await processPackageJob(jobs, job.id, prepare)).toBe("retry"); - expect(await processPackageJob(jobs, job.id, prepare)).toBe("retry"); - expect(await processPackageJob(jobs, job.id, prepare)).toBe("ack"); + expect(await processPackagingJob(jobs, job.id, prepare)).toBe("retry"); + expect(await processPackagingJob(jobs, job.id, prepare)).toBe("retry"); + expect(await processPackagingJob(jobs, job.id, prepare)).toBe("ack"); expect(job.status).toBe("failed"); expect(job.error?.message).not.toContain("storage unavailable"); expect(logged).toHaveBeenCalledWith( @@ -130,7 +130,7 @@ describe("provider-independent package job processing", () => { for (const status of [422, 403] as const) { const { jobs, job } = repository(); expect( - await processPackageJob(jobs, job.id, async () => { + await processPackagingJob(jobs, job.id, async () => { throw new PackageError("REJECTED", "Rejected", status); }) ).toBe("ack"); diff --git a/test/services/extensions/v2/pending-edits.test.ts b/services/api/test/services/extensions/v2/pending-edits.test.ts similarity index 100% rename from test/services/extensions/v2/pending-edits.test.ts rename to services/api/test/services/extensions/v2/pending-edits.test.ts diff --git a/test/services/extensions/v2/public-extensions.test.ts b/services/api/test/services/extensions/v2/public-extensions.test.ts similarity index 100% rename from test/services/extensions/v2/public-extensions.test.ts rename to services/api/test/services/extensions/v2/public-extensions.test.ts diff --git a/test/services/extensions/v2/publisher.test.ts b/services/api/test/services/extensions/v2/publisher.test.ts similarity index 100% rename from test/services/extensions/v2/publisher.test.ts rename to services/api/test/services/extensions/v2/publisher.test.ts diff --git a/test/services/extensions/v2/resource-bounds.test.ts b/services/api/test/services/extensions/v2/resource-bounds.test.ts similarity index 99% rename from test/services/extensions/v2/resource-bounds.test.ts rename to services/api/test/services/extensions/v2/resource-bounds.test.ts index 35d91192..0a27a633 100644 --- a/test/services/extensions/v2/resource-bounds.test.ts +++ b/services/api/test/services/extensions/v2/resource-bounds.test.ts @@ -5,7 +5,7 @@ import { waitOnExecutionContext } from "cloudflare:test"; import { env } from "cloudflare:workers"; -import app from "../../../../src/app"; +import app from "../../../../src/app/index"; import { getExtensionsDb } from "../../../../src/lib/db"; import { maintainExtensionResources, diff --git a/services/api/test/services/extensions/v2/revalidate.node.test.ts b/services/api/test/services/extensions/v2/revalidate.node.test.ts new file mode 100644 index 00000000..edc85b06 --- /dev/null +++ b/services/api/test/services/extensions/v2/revalidate.node.test.ts @@ -0,0 +1,53 @@ +import { expect, it } from "vitest"; +import { Hono } from "hono"; +import { createMemoryCache } from "../../../../src/lib/adapters/node/cache"; +import { platformMiddleware } from "../../../../src/lib/middleware"; +import { listCacheKey } from "../../../../src/services/extensions/v1/list-cache"; +import { revalidateExtensions } from "../../../../src/services/extensions/v2/revalidate"; + +it("revalidates through the platform capabilities without a Cloudflare context", async () => { + const cache = createMemoryCache(); + const tasks: Promise[] = []; + const requests: Request[] = []; + const app = new Hono(); + app.use( + "*", + platformMiddleware( + { + caches: { CACHE_KV: cache }, + environment: { + get: (name) => + name === "EXTENSIONS_REVALIDATE_SECRET" ? "local-secret" : undefined + }, + services: { + EXTENSIONS_FRONTEND: async (request) => { + requests.push(request); + return new Response(null, { status: 204 }); + } + } + }, + (task) => tasks.push(task) + ) + ); + app.post("/revalidate", (c) => { + revalidateExtensions(c); + return c.body(null, 204); + }); + + try { + const key = listCacheKey("module", undefined); + await cache.put(key, "cached extension list"); + await cache.put("unrelated", "keep"); + expect((await app.request("/revalidate", { method: "POST" })).status).toBe( + 204 + ); + expect(tasks.length).toBeGreaterThan(0); + await Promise.all(tasks); + expect(await cache.get(key)).toBeNull(); + expect(await cache.get("unrelated")).toBe("keep"); + expect(requests).toHaveLength(1); + expect(requests[0]).toBeInstanceOf(Request); + } finally { + cache.close(); + } +}); diff --git a/test/services/previews/v1/budget.test.ts b/services/api/test/services/previews/v1/budget.test.ts similarity index 99% rename from test/services/previews/v1/budget.test.ts rename to services/api/test/services/previews/v1/budget.test.ts index 48d0cfbd..02654372 100644 --- a/test/services/previews/v1/budget.test.ts +++ b/services/api/test/services/previews/v1/budget.test.ts @@ -5,7 +5,7 @@ import { runInDurableObject, waitOnExecutionContext } from "cloudflare:test"; -import app from "../../../../src/app"; +import app from "../../../../src/app/index"; import { Hono } from "hono"; import { previewGitHub, diff --git a/test/services/previews/v1/cache.test.ts b/services/api/test/services/previews/v1/cache.test.ts similarity index 100% rename from test/services/previews/v1/cache.test.ts rename to services/api/test/services/previews/v1/cache.test.ts diff --git a/test/services/previews/v1/commit.test.ts b/services/api/test/services/previews/v1/commit.test.ts similarity index 99% rename from test/services/previews/v1/commit.test.ts rename to services/api/test/services/previews/v1/commit.test.ts index 467545a3..cb00c077 100644 --- a/test/services/previews/v1/commit.test.ts +++ b/services/api/test/services/previews/v1/commit.test.ts @@ -5,7 +5,7 @@ import { waitOnExecutionContext } from "cloudflare:test"; import { env } from "cloudflare:workers"; -import app from "../../../../src/app"; +import app from "../../../../src/app/index"; import { MockGitHubRequest } from "../../../utils/test-types"; import { suppressConsole } from "../../../utils/mock-helpers"; diff --git a/test/services/previews/v1/main.test.ts b/services/api/test/services/previews/v1/main.test.ts similarity index 99% rename from test/services/previews/v1/main.test.ts rename to services/api/test/services/previews/v1/main.test.ts index 0ad538d6..0c3e605a 100644 --- a/test/services/previews/v1/main.test.ts +++ b/services/api/test/services/previews/v1/main.test.ts @@ -5,7 +5,7 @@ import { waitOnExecutionContext } from "cloudflare:test"; import { env } from "cloudflare:workers"; -import app from "../../../../src/app"; +import app from "../../../../src/app/index"; import { MockGitHubRequest } from "../../../utils/test-types"; import { suppressConsole } from "../../../utils/mock-helpers"; diff --git a/test/services/previews/v1/pr.test.ts b/services/api/test/services/previews/v1/pr.test.ts similarity index 99% rename from test/services/previews/v1/pr.test.ts rename to services/api/test/services/previews/v1/pr.test.ts index c077cda7..dc4baefa 100644 --- a/test/services/previews/v1/pr.test.ts +++ b/services/api/test/services/previews/v1/pr.test.ts @@ -5,7 +5,7 @@ import { waitOnExecutionContext } from "cloudflare:test"; import { env } from "cloudflare:workers"; -import app from "../../../../src/app"; +import app from "../../../../src/app/index"; import { MockGitHubRequest } from "../../../utils/test-types"; import { suppressConsole } from "../../../utils/mock-helpers"; diff --git a/test/services/stats/v1/index.test.ts b/services/api/test/services/stats/v1/index.test.ts similarity index 99% rename from test/services/stats/v1/index.test.ts rename to services/api/test/services/stats/v1/index.test.ts index d2b399e4..2bf89d6a 100644 --- a/test/services/stats/v1/index.test.ts +++ b/services/api/test/services/stats/v1/index.test.ts @@ -5,7 +5,7 @@ import { createExecutionContext, waitOnExecutionContext } from "cloudflare:test"; -import app from "../../../../src/app"; +import app from "../../../../src/app/index"; import { mockGitHubReleases, mockComposerJson, diff --git a/test/services/versions/v1/errors.test.ts b/services/api/test/services/versions/v1/errors.test.ts similarity index 99% rename from test/services/versions/v1/errors.test.ts rename to services/api/test/services/versions/v1/errors.test.ts index 80b47887..c58db157 100644 --- a/test/services/versions/v1/errors.test.ts +++ b/services/api/test/services/versions/v1/errors.test.ts @@ -5,7 +5,7 @@ import { waitOnExecutionContext } from "cloudflare:test"; import { env } from "cloudflare:workers"; -import app from "../../../../src/app"; +import app from "../../../../src/app/index"; import { mockGitHubReleases, mockComposerJson diff --git a/test/services/versions/v1/index.test.ts b/services/api/test/services/versions/v1/index.test.ts similarity index 99% rename from test/services/versions/v1/index.test.ts rename to services/api/test/services/versions/v1/index.test.ts index 5eb02bb4..f166eef3 100644 --- a/test/services/versions/v1/index.test.ts +++ b/services/api/test/services/versions/v1/index.test.ts @@ -6,7 +6,7 @@ import { waitOnExecutionContext } from "cloudflare:test"; import { env } from "cloudflare:workers"; -import app from "../../../../src/app"; +import app from "../../../../src/app/index"; import { mockGitHubReleases, diff --git a/test/services/versions/v1/middleware.test.ts b/services/api/test/services/versions/v1/middleware.test.ts similarity index 98% rename from test/services/versions/v1/middleware.test.ts rename to services/api/test/services/versions/v1/middleware.test.ts index fbf981f6..633b5170 100644 --- a/test/services/versions/v1/middleware.test.ts +++ b/services/api/test/services/versions/v1/middleware.test.ts @@ -5,7 +5,7 @@ import { waitOnExecutionContext } from "cloudflare:test"; import { env } from "cloudflare:workers"; -import app from "../../../../src/app"; +import app from "../../../../src/app/index"; import { mockGitHubReleases, mockComposerJson diff --git a/test/utils/apply-migrations.ts b/services/api/test/utils/apply-migrations.ts similarity index 100% rename from test/utils/apply-migrations.ts rename to services/api/test/utils/apply-migrations.ts diff --git a/test/utils/isolate-edge-cache.ts b/services/api/test/utils/isolate-edge-cache.ts similarity index 100% rename from test/utils/isolate-edge-cache.ts rename to services/api/test/utils/isolate-edge-cache.ts diff --git a/test/utils/mock-helpers.ts b/services/api/test/utils/mock-helpers.ts similarity index 100% rename from test/utils/mock-helpers.ts rename to services/api/test/utils/mock-helpers.ts diff --git a/test/utils/test-types.ts b/services/api/test/utils/test-types.ts similarity index 100% rename from test/utils/test-types.ts rename to services/api/test/utils/test-types.ts diff --git a/services/api/tsconfig.json b/services/api/tsconfig.json new file mode 100644 index 00000000..b4672532 --- /dev/null +++ b/services/api/tsconfig.json @@ -0,0 +1,11 @@ +{ + "extends": "../../tsconfig.json", + "include": ["src/**/*.ts", "test/**/*.ts", "*.ts", "scripts/**/*.ts"], + "compilerOptions": { + "types": [ + "./worker-configuration.d.ts", + "@cloudflare/vitest-plugin", + "node" + ] + } +} diff --git a/vitest.config.ts b/services/api/vitest.config.ts similarity index 98% rename from vitest.config.ts rename to services/api/vitest.config.ts index 0fd362d1..b2beaf96 100644 --- a/vitest.config.ts +++ b/services/api/vitest.config.ts @@ -18,7 +18,6 @@ export default defineConfig({ cloudflareTest({ wrangler: { configPath: "./wrangler.jsonc" }, miniflare: { - images: { binding: "PACKAGE_IMAGES" }, // Add test environment variables bindings: { GITHUB_TOKEN: "test-github-token", diff --git a/vitest.node.config.ts b/services/api/vitest.node.config.ts similarity index 89% rename from vitest.node.config.ts rename to services/api/vitest.node.config.ts index 11e93c0b..22bdcf76 100644 --- a/vitest.node.config.ts +++ b/services/api/vitest.node.config.ts @@ -6,7 +6,7 @@ export default defineConfig({ include: [ "test/lib/adapters/node/**/*.test.ts", "test/services/extensions/v2/migrations.test.ts", - "test/services/extensions/v2/*.node.test.ts" + "test/**/*.node.test.ts" ], testTimeout: 10000, diff --git a/worker-configuration.d.ts b/services/api/worker-configuration.d.ts similarity index 99% rename from worker-configuration.d.ts rename to services/api/worker-configuration.d.ts index 5e25d541..ea247648 100644 --- a/worker-configuration.d.ts +++ b/services/api/worker-configuration.d.ts @@ -1,5 +1,5 @@ /* eslint-disable */ -// Generated by Wrangler by running `wrangler types --env-interface=CloudflareBindings` (hash: e610663ea57a18e0636f4e14dece74ae) +// Generated by Wrangler by running `wrangler types --env-interface=CloudflareBindings` (hash: d1888a571fcbd45ad7ca22833c048474) // Runtime types generated with workerd@1.20261104.1 2026-10-01 interface __BaseEnv_CloudflareBindings { AUTH_KV: KVNamespace; @@ -10,7 +10,6 @@ interface __BaseEnv_CloudflareBindings { EXTENSION_PACKAGE_QUEUE: Queue; EXTENSION_WRITE_RATE_LIMITER: RateLimit; PROFILE_CREATION_RATE_LIMITER: RateLimit; - PACKAGE_IMAGES: ImagesBinding; ENVIRONMENT: "production"; EXTENSION_PACKAGES_MODE: "disabled"; EXTENSION_PACKAGES_BASE_URL: "https://api.fossbilling.net/extensions/v2"; diff --git a/wrangler.jsonc b/services/api/wrangler.jsonc similarity index 79% rename from wrangler.jsonc rename to services/api/wrangler.jsonc index 9ea8ef5c..465eab43 100644 --- a/wrangler.jsonc +++ b/services/api/wrangler.jsonc @@ -1,5 +1,5 @@ { - "$schema": "node_modules/wrangler/config-schema.json", + "$schema": "../../node_modules/wrangler/config-schema.json", "name": "api", "main": "src/app/index.ts", "compatibility_date": "2026-10-01", @@ -24,7 +24,7 @@ "keep_vars": true, "vars": { "ENVIRONMENT": "production", - // Enable only after the queue, Images binding and B2 secrets are configured. + // Enable only after the queue, HTTP processor and B2 secrets are configured. "EXTENSION_PACKAGES_MODE": "disabled", "EXTENSION_PACKAGES_BASE_URL": "https://api.fossbilling.net/extensions/v2", "EXTENSION_ASSETS_PUBLIC_URL": "https://extensions-assets.fossbilling.net", @@ -77,7 +77,6 @@ } ] }, - "images": { "binding": "PACKAGE_IMAGES", "remote": false }, "r2_buckets": [ { "binding": "DOWNLOAD_BUCKET", @@ -119,5 +118,30 @@ "period": 60 } } - ] + ], + "previews": { + "vars": { + "ENVIRONMENT": "preview", + "EXTENSION_PACKAGES_MODE": "disabled", + "EXTENSIONS_V2_EMAIL_PROVIDER": "disabled", + "EXTENSIONS_RETENTION_MODE": "dry-run" + }, + "durable_objects": { + "bindings": [ + { "name": "PREVIEW_GITHUB_BUDGET", "class_name": "PreviewGitHubBudget" } + ] + }, + "ratelimits": [ + { + "name": "EXTENSION_WRITE_RATE_LIMITER", + "namespace_id": "2002", + "simple": { "limit": 60, "period": 60 } + }, + { + "name": "PROFILE_CREATION_RATE_LIMITER", + "namespace_id": "2001", + "simple": { "limit": 3, "period": 60 } + } + ] + } } diff --git a/services/extension-packager/README.md b/services/extension-packager/README.md new file mode 100644 index 00000000..bf0b01a5 --- /dev/null +++ b/services/extension-packager/README.md @@ -0,0 +1,108 @@ +# Extension packager + +A separately deployed Node service for ZIP validation, manifest replacement and icon +preparation. It imports the shared `@fossbilling/extension-packaging` contracts and +implementation; it has no dependency on the API service source, database or Cloudflare +bindings. Netlify's entry point in `netlify/functions/extension-packager.ts` delegates +to the same standard Request/Response handler used locally in `src/handler.ts`. + +## Local development + +Install dependencies with `npm ci` from the repository root. Put development settings +in the ignored `services/extension-packager/.dev.vars` file. It needs: + +```dotenv +EXTENSION_PACKAGER_SECRET=local-shared-secret +EXTENSION_PACKAGES_S3_ENDPOINT=https://s3.example.invalid +EXTENSION_PACKAGES_S3_REGION=development-region +EXTENSION_PACKAGES_S3_BUCKET=development-bucket +EXTENSION_PACKAGES_S3_PREFIX=local +EXTENSION_PACKAGES_S3_ACCESS_KEY_ID=development-key +EXTENSION_PACKAGES_S3_SECRET_ACCESS_KEY=development-secret +``` + +Use the actual development bucket settings locally, matching the API's private bucket +and object prefix. Configure `services/api/.dev.vars` with +`EXTENSION_PACKAGER_URL=http://127.0.0.1:8790/process` and the same shared secret. +The public asset bucket credentials remain configured only on the API. + +From the repository root: + +```sh +npm run extension-packager:dev +npm test --workspace @fossbilling/extension-packager +``` + +The server listens on `127.0.0.1:8790`; `EXTENSION_PACKAGER_PORT` can override the port. +Tests use simulated S3 responses and do not touch live B2 buckets. + +## Netlify deployment + +Create a dedicated Netlify project connected to this repository: + +- Base directory: leave unset to install from the repository root and its lockfile. +- Package directory: `services/extension-packager`. +- Build command: leave blank; Netlify bundles the function automatically. +- Configuration: `services/extension-packager/netlify.toml`. +- Publish directory: `services/extension-packager/netlify/public` (empty placeholder). +- Functions directory: `services/extension-packager/netlify/functions`. + +Paths in `netlify.toml` are relative to the repository-root base directory, following +[Netlify's monorepo configuration](https://docs.netlify.com/build/configure-builds/monorepos/). +The Netlify configuration currently selects Node.js 24, a stable Lambda runtime. +[AWS lists Node.js 26 as public preview](https://docs.aws.amazon.com/lambda/latest/dg/lambda-runtimes.html), +so it is not yet the production hosting target. The code is tested on Node.js 26; +this runtime selection belongs only to the Netlify adapter. + +Native Sharp and sanitizer dependencies remain external so Netlify includes their +binaries and WASM files. Verify those files and the deployed runtime before +enabling processing. + +Set the seven variables listed above through Netlify's dashboard, CLI or API for the +function runtime. Use production credentials and a separate randomly generated shared +secret for production. Keep production values in the production deploy context. +Do not provide public asset bucket credentials, D1 credentials or API administration +tokens. Netlify runtime variables do not come from `netlify.toml`; redeploy after changing them. + +After deployment, set the API's dashboard-managed `EXTENSION_PACKAGER_URL` to +`https://.netlify.app/.netlify/functions/extension-packager`, and set its +`EXTENSION_PACKAGER_SECRET` to the same secret. The HTTP adapter requires +HTTPS in production and allows loopback HTTP for local development. + +Deploying this project only deploys the packager and its imported code. It does not +start or deploy the API. A Git-connected project can redeploy its configured branch +independently of the API's GitHub Actions deployment. + +## Processing and recovery + +The API retains D1 ownership checks, job leases, immutable source/artifact records, +and publication. ZIP validation, manifest replacement and image preparation run +outside Workers Free. The API has no inline-processing fallback. + +The API reserves bounded private B2 output keys before invoking `PackageProcessor`. +The runner reads the source and writes the resulting ZIP and temporary prepared +icons directly to the private package bucket. Requests and responses contain only +metadata, not ZIP bytes, avoiding function request/response size limits. +The API then creates/reuses the permanent extension draft, copies the prepared +icons to their public UUID paths, records the artifact, and deletes temporary icons. +No ZIP is reopened to serve an icon. The runner needs no D1 credentials or public +asset bucket key. + +Output reservations use the maximum object size until processing succeeds; the +archive reservation then shrinks to its actual size. Unused icon reservations are +removed after successful publication to the asset bucket. Failed or interrupted +outputs remain tracked for normal cleanup, including a runner whose response was +lost after it wrote to B2. + +This is a regular synchronous function. The existing API Queue provides background +delivery and waits for the completed response before acknowledging the job. Do not +use Netlify background-function mode, which returns 202 before processing completes. +The packager deadline is 45 seconds, the HTTP client timeout is 50 seconds, and the +Netlify synchronous limit is 60 seconds. Invalid ZIPs and icons return 422; transport +failures, timeouts and unavailable storage return retryable 503 errors. Existing API +attempt limits and lease recovery apply. + +Production package processing remains disabled in `services/api/wrangler.jsonc`. +Before enablement, verify the Netlify deployment, measure runtime and credit usage, +and repeat the controlled production package test, including near-limit ZIPs, +SVG/raster icons, stored-source rebuilds, approval and cleanup. diff --git a/services/extension-packager/netlify.toml b/services/extension-packager/netlify.toml new file mode 100644 index 00000000..2800c5c4 --- /dev/null +++ b/services/extension-packager/netlify.toml @@ -0,0 +1,11 @@ +[build] + publish = "services/extension-packager/netlify/public" + +[build.environment] + # Lambda Node 26 is currently public preview; application CI uses Node 26. + NODE_VERSION = "24" + +[functions] + directory = "services/extension-packager/netlify/functions" + node_bundler = "esbuild" + external_node_modules = ["sharp", "@zeropress/svg-hush"] diff --git a/services/extension-packager/netlify/functions/extension-packager.ts b/services/extension-packager/netlify/functions/extension-packager.ts new file mode 100644 index 00000000..b1c4af11 --- /dev/null +++ b/services/extension-packager/netlify/functions/extension-packager.ts @@ -0,0 +1,4 @@ +import { handlePackageProcessing } from "../../src/handler"; + +export default (request: Request) => + handlePackageProcessing(request, (name) => process.env[name]); diff --git a/services/extension-packager/netlify/public/.gitkeep b/services/extension-packager/netlify/public/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/services/extension-packager/package.json b/services/extension-packager/package.json new file mode 100644 index 00000000..00ba1210 --- /dev/null +++ b/services/extension-packager/package.json @@ -0,0 +1,22 @@ +{ + "name": "@fossbilling/extension-packager", + "private": true, + "version": "0.0.0", + "type": "module", + "license": "AGPL-3.0-only", + "scripts": { + "dev": "tsx --env-file=.dev.vars scripts/extension-packager-dev.ts", + "test": "vitest run", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "@fossbilling/extension-packaging": "*", + "@fossbilling/logging": "*", + "@zeropress/svg-hush": "0.1.1", + "sharp": "0.35.5" + }, + "devDependencies": { + "tsx": "4.23.15", + "@zip.js/zip.js": "2.24.0" + } +} diff --git a/services/extension-packager/scripts/extension-packager-dev.ts b/services/extension-packager/scripts/extension-packager-dev.ts new file mode 100644 index 00000000..20db0ed2 --- /dev/null +++ b/services/extension-packager/scripts/extension-packager-dev.ts @@ -0,0 +1,37 @@ +import { logError } from "@fossbilling/logging"; +import { createServer } from "node:http"; +import { Readable } from "node:stream"; +import { handlePackageProcessing } from "../src/handler"; + +const port = Number(process.env.EXTENSION_PACKAGER_PORT || 8790); +createServer(async (incoming, outgoing) => { + const controller = new AbortController(); + outgoing.on("close", () => { + if (!outgoing.writableFinished) controller.abort(); + }); + try { + const init = { + method: incoming.method, + headers: incoming.headers as Record, + body: ["GET", "HEAD"].includes(incoming.method ?? "GET") + ? undefined + : Readable.toWeb(incoming), + signal: controller.signal, + duplex: "half" as const + }; + const response = await handlePackageProcessing( + new Request(`http://localhost:${port}${incoming.url}`, init), + (name) => process.env[name] + ); + outgoing.writeHead(response.status, Object.fromEntries(response.headers)); + outgoing.end(new Uint8Array(await response.arrayBuffer())); + } catch (error) { + logError("extensions/packages", "Local packager request failed", { + error: error instanceof Error ? error.message : "Unknown error" + }); + outgoing.writeHead(500); + outgoing.end(); + } +}).listen(port, "127.0.0.1", () => + console.log(`Extension packager listening on http://127.0.0.1:${port}`) +); diff --git a/services/extension-packager/src/handler.ts b/services/extension-packager/src/handler.ts new file mode 100644 index 00000000..80836d17 --- /dev/null +++ b/services/extension-packager/src/handler.ts @@ -0,0 +1,75 @@ +import { logError } from "@fossbilling/logging"; +import { timingSafeEqual } from "node:crypto"; +import { b2ObjectStore } from "@fossbilling/extension-packaging/adapters/b2"; +import { NodePackageImages } from "./images"; +import { ProcessingInputSchema } from "@fossbilling/extension-packaging/processor"; +import { processPackage } from "@fossbilling/extension-packaging/process"; +import { PackageError } from "@fossbilling/extension-packaging/errors"; +import { boundedBytes } from "@fossbilling/extension-packaging/store"; + +// Standard Request/Response keeps the handler usable on Netlify or another Node host. +export async function handlePackageProcessing( + request: Request, + getEnv: (name: string) => string | undefined +): Promise { + const headers = { "Cache-Control": "no-store" }; + const secret = getEnv("EXTENSION_PACKAGER_SECRET"); + if (!secret) + return Response.json( + { error: { message: "Package processor is not configured" } }, + { status: 503, headers } + ); + const supplied = Buffer.from(request.headers.get("Authorization") ?? ""); + const expected = Buffer.from(`Bearer ${secret}`); + if ( + supplied.length !== expected.length || + !timingSafeEqual(supplied, expected) + ) + return new Response(null, { status: 401, headers }); + if (request.method !== "POST") + return new Response(null, { + status: 405, + headers: { ...headers, Allow: "POST" } + }); + const signal = AbortSignal.any([request.signal, AbortSignal.timeout(45000)]); + try { + // A 64 KiB manifest string can expand in UTF-8, alongside request metadata. + const bytes = await boundedBytes(request.body, 256 * 1024); + let input; + try { + input = ProcessingInputSchema.parse( + JSON.parse(new TextDecoder().decode(bytes)) + ); + } catch { + return Response.json( + { error: { message: "Invalid processing request" } }, + { status: 422, headers } + ); + } + const result = await processPackage( + input, + b2ObjectStore(getEnv, "packages", signal), + new NodePackageImages(), + signal + ); + return Response.json({ result }, { headers }); + } catch (error) { + if (error instanceof PackageError && [413, 422].includes(error.status)) + return Response.json( + { error: { code: error.code, message: error.message } }, + { status: 422, headers } + ); + if (error instanceof PackageError && error.code === "SOURCE_UNAVAILABLE") + return Response.json( + { error: { code: error.code, message: error.message } }, + { status: 503, headers } + ); + logError("extensions/packages", "Package processing failed", { + error: error instanceof Error ? error.message : "Unknown error" + }); + return Response.json( + { error: { message: "Package processing is temporarily unavailable" } }, + { status: 503, headers } + ); + } +} diff --git a/services/extension-packager/src/images.ts b/services/extension-packager/src/images.ts new file mode 100644 index 00000000..e30d1113 --- /dev/null +++ b/services/extension-packager/src/images.ts @@ -0,0 +1,61 @@ +import sharp from "sharp"; +import { sanitizeSvg } from "@zeropress/svg-hush/node"; +import type { + PackageImageProcessor, + IconVariant +} from "@fossbilling/extension-packaging/images"; +import { PackageError } from "@fossbilling/extension-packaging/errors"; + +export class NodePackageImages implements PackageImageProcessor { + async prepare( + icon: Uint8Array, + isSvg: boolean + ): Promise { + try { + if ( + isSvg || + (await sharp(icon, { limitInputPixels: 16 * 1024 * 1024 }).metadata()) + .format === "svg" + ) + return [{ format: "svg", bytes: new Uint8Array(sanitizeSvg(icon)) }]; + const variants: IconVariant[] = []; + for (const format of ["avif", "png"] as const) { + const bytes = new Uint8Array( + await sharp(icon, { limitInputPixels: 16 * 1024 * 1024 }) + .rotate() + .resize({ + width: 256, + height: 256, + fit: "inside", + withoutEnlargement: true + }) + .toFormat(format) + .timeout({ seconds: 10 }) + .toBuffer() + ); + if (bytes.length > 1024 * 1024) + throw new Error("Prepared icon exceeds size limit"); + variants.push({ format, bytes }); + } + return variants; + } catch (error) { + // libvips reports processing deadlines and allocation failures as errors. + if ( + error instanceof Error && + /timeout:|out of memory|could not allocate|unable to allocate/i.test( + error.message + ) + ) + throw new PackageError( + "PACKAGE_UNAVAILABLE", + "Image processing is temporarily unavailable", + 503 + ); + throw new PackageError( + "INVALID_ICON", + "Package icon could not be processed", + 422 + ); + } + } +} diff --git a/services/extension-packager/test/processor.test.ts b/services/extension-packager/test/processor.test.ts new file mode 100644 index 00000000..c7c5ca50 --- /dev/null +++ b/services/extension-packager/test/processor.test.ts @@ -0,0 +1,396 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + TextReader, + TextWriter, + Uint8ArrayReader, + Uint8ArrayWriter, + ZipReader, + ZipWriter +} from "@zip.js/zip.js"; +import sharp from "sharp"; +import { createServer } from "node:http"; +import { once } from "node:events"; +import { handlePackageProcessing } from "../src/handler"; +import { HttpPackageProcessor } from "@fossbilling/extension-packaging/adapters/http"; +import { sha256 } from "@fossbilling/extension-packaging/checksum"; +import type { ProcessingInput } from "@fossbilling/extension-packaging/processor"; + +const encode = (text: string) => new Uint8Array(new TextEncoder().encode(text)); + +const manifest = { + manifest_version: 1 as const, + id: "processor-example", + type: "module" as const, + name: "Processor Example", + version: "1.0.0", + description: "Local processor test", + author: { name: "Developer" }, + license: "MIT", + requires: { fossbilling: ">=0.7.0", php: ">=8.3" } +}; +const config: Record = { + EXTENSION_PACKAGER_SECRET: "local-test-secret", + EXTENSION_PACKAGES_S3_ENDPOINT: "https://s3.example.test", + EXTENSION_PACKAGES_S3_REGION: "test-region", + EXTENSION_PACKAGES_S3_BUCKET: "private-packages", + EXTENSION_PACKAGES_S3_PREFIX: "test", + EXTENSION_PACKAGES_S3_ACCESS_KEY_ID: "test-key", + EXTENSION_PACKAGES_S3_SECRET_ACCESS_KEY: "test-secret" +}; +const options = { useWebWorkers: false, useCompressionStream: true }; +const svg = ` + +
HTML
+ + +
`; +afterEach(() => { + vi.unstubAllGlobals(); + vi.restoreAllMocks(); +}); +async function archive(iconName = "icon.svg", icon = encode(svg)) { + const writer = new ZipWriter(new Uint8ArrayWriter(), options); + await writer.add("manifest.json", new TextReader("replaced metadata")); + await writer.add( + "Service.php", + new TextReader("; checksum: string } + >(); + const key = (id: string) => "/private-packages/test/packages/" + id; + const handler = (request: Request) => + handlePackageProcessing(request, (name) => config[name]); + const fetch = vi.fn(async (request: Request) => { + const url = new URL(request.url); + if (url.hostname === "processor.example.test") return handler(request); + if (url.hostname === "127.0.0.1") return nativeFetch(request); + if (url.hostname !== "s3.example.test") + throw new Error("Unexpected network request"); + expect(request.cache).toBe("no-store"); + if (request.method === "PUT") { + objects.set(url.pathname, { + bytes: new Uint8Array(await request.arrayBuffer()), + checksum: request.headers.get("x-amz-meta-sha256")! + }); + return new Response(null); + } + const value = objects.get(url.pathname); + return value + ? new Response(value.bytes, { + headers: { "x-amz-meta-sha256": value.checksum } + }) + : new Response(null, { status: 404 }); + }); + vi.stubGlobal("fetch", fetch); + const processor = new HttpPackageProcessor( + "https://processor.example.test/process", + config.EXTENSION_PACKAGER_SECRET + ); + const input = async ( + bytes: Uint8Array + ): Promise => { + const id = crypto.randomUUID(), + checksum = await sha256(bytes); + objects.set(key(id), { bytes, checksum }); + return { + manifest: structuredClone(manifest), + source: { id, sha256: checksum }, + archive: crypto.randomUUID(), + icons: [crypto.randomUUID(), crypto.randomUUID()] + }; + }; + return { objects, key, fetch, handler, processor, input }; +} +describe("Node package processor over the generic HTTP contract", () => { + it("stores the prepared ZIP privately, sanitizes SVG, and rebuilds from the same original source", async () => { + const f = fixture(), + input = await f.input(await archive()); + const result = await f.processor.process(input); + const bytes = f.objects.get(f.key(input.archive))!.bytes; + expect(await sha256(bytes)).toBe(result.archive.sha256); + const reader = new ZipReader(new Uint8ArrayReader(bytes), options); + const files: Record = {}; + for (const entry of await reader.getEntries()) + if (!entry.directory) + files[entry.filename] = await entry.getData(new TextWriter()); + await reader.close(); + expect(JSON.parse(files["manifest.json"])).toEqual(manifest); + expect(files["Service.php"]).toBe( + " { + const f = fixture(); + const original = new Uint8Array( + await sharp({ + create: { + width: 512, + height: 256, + channels: 4, + background: "#2364ff80" + } + }) + .toFormat(format) + .toBuffer() + ); + const result = await f.processor.process( + await f.input(await archive("icon." + format, original)) + ); + expect(result.icons.map((i) => i.format)).toEqual(["avif", "png"]); + for (const icon of result.icons) { + const bytes = f.objects.get(f.key(icon.id))!.bytes; + const info = await sharp(bytes).metadata(); + expect(info).toMatchObject({ + format: icon.format === "avif" ? "heif" : "png", + width: 256, + height: 128 + }); + expect(await sha256(bytes)).toBe(icon.sha256); + } + } + ); + it("sanitizes SVG masquerading as a raster icon and rejects invalid images before writing outputs", async () => { + const f = fixture(); + const result = await f.processor.process( + await f.input(await archive("icon.png")) + ); + expect(result.icons[0].format).toBe("svg"); + const sanitized = f.objects.get(f.key(result.icons[0].id))!.bytes; + expect(new TextDecoder().decode(sanitized)).not.toMatch(/')], + [ + "icon.svg", + encode( + '" + ) + ], + ["icon.png", encode("not an image")] + ] as const) { + const input = await f.input(await archive(name, bytes)); + await expect(f.processor.process(input)).rejects.toMatchObject({ + code: "INVALID_ICON", + status: 422 + }); + expect(f.objects.has(f.key(input.archive))).toBe(false); + } + }); + it("accepts a valid manifest whose UTF-8 processing request exceeds 64 KiB", async () => { + const f = fixture(); + const input = await f.input(await archive()); + input.manifest.requires.modules = Object.fromEntries( + Array.from({ length: 30 }, (_, i) => [ + `dependency_${i}`, + ">=1.0.0" + "\u2003".repeat(990) + ]) + ); + expect(JSON.stringify(input.manifest).length).toBeLessThan(64 * 1024); + expect(Buffer.byteLength(JSON.stringify(input))).toBeGreaterThan(64 * 1024); + await expect(f.processor.process(input)).resolves.toMatchObject({ + archive: { bytes: expect.any(Number) } + }); + }); + it.each([ + "timeout: 10% complete", + "Could not allocate enough memory for raw output" + ])("keeps image processing failure retryable: %s", async (message) => { + const f = fixture(); + const png = new Uint8Array( + await sharp({ + create: { width: 1, height: 1, channels: 4, background: "red" } + }) + .png() + .toBuffer() + ); + const input = await f.input(await archive("icon.png", png)); + vi.spyOn(sharp.prototype, "toBuffer").mockRejectedValueOnce( + new Error(message) + ); + vi.spyOn(console, "error").mockImplementation(() => {}); + await expect(f.processor.process(input)).rejects.toMatchObject({ + status: 503 + }); + expect(f.objects.has(f.key(input.archive))).toBe(false); + }); + it("preserves a missing source error through the HTTP processor", async () => { + const f = fixture(); + const input = await f.input(await archive()); + f.objects.delete(f.key(input.source.id)); + await expect(f.processor.process(input)).rejects.toMatchObject({ + code: "SOURCE_UNAVAILABLE", + status: 503 + }); + expect(f.objects.has(f.key(input.archive))).toBe(false); + }); + it("rejects unauthenticated and invalid requests before storage access", async () => { + const f = fixture(); + expect( + ( + await f.handler( + new Request("https://processor.example.test", { + method: "POST", + body: "{}" + }) + ) + ).status + ).toBe(401); + expect(f.fetch).not.toHaveBeenCalled(); + const response = await f.handler( + new Request("https://processor.example.test", { + method: "POST", + headers: { + Authorization: "Bearer " + config.EXTENSION_PACKAGER_SECRET + }, + body: "{}" + }) + ); + expect(response.status).toBe(422); + expect(f.fetch).not.toHaveBeenCalled(); + }); + it("rejects invalid archives and changed source bytes without writing outputs", async () => { + const f = fixture(), + input = await f.input(encode("not a ZIP")); + await expect(f.processor.process(input)).rejects.toMatchObject({ + code: "INVALID_PACKAGE", + status: 422 + }); + expect(f.objects.size).toBe(1); + const valid = await f.input(await archive()); + f.objects.get(f.key(valid.source.id))!.bytes = new Uint8Array([1, 2, 3]); + vi.spyOn(console, "error").mockImplementation(() => {}); + await expect(f.processor.process(valid)).rejects.toMatchObject({ + status: 503 + }); + expect(f.objects.has(f.key(valid.archive))).toBe(false); + }); + it("processes a near-limit ZIP over a real local HTTP connection with a small JSON response", async () => { + const f = fixture(); + const writer = new ZipWriter(new Uint8ArrayWriter(), options); + for (let i = 0; i < 500; i++) { + const bytes = new Uint8Array(33000); + crypto.getRandomValues(bytes.subarray(0, 9900)); + await writer.add(`files/file-${i}.bin`, new Uint8ArrayReader(bytes)); + } + const source = await writer.close(); + expect(source.length).toBeLessThanOrEqual(5 * 1024 * 1024); + expect(source.length).toBeGreaterThan(4.5 * 1024 * 1024); + const input = await f.input(source); + const server = createServer(async (req, res) => { + const chunks: Buffer[] = []; + for await (const chunk of req) chunks.push(Buffer.from(chunk)); + const response = await f.handler( + new Request("http://localhost/process", { + method: "POST", + headers: { Authorization: String(req.headers.authorization) }, + body: Buffer.concat(chunks) + }) + ); + res.writeHead(response.status, Object.fromEntries(response.headers)); + res.end(new Uint8Array(await response.arrayBuffer())); + }); + server.listen(0, "127.0.0.1"); + await once(server, "listening"); + try { + const address = server.address(); + if (!address || typeof address === "string") + throw new Error("Missing HTTP port"); + const processor = new HttpPackageProcessor( + `http://127.0.0.1:${address.port}/process`, + config.EXTENSION_PACKAGER_SECRET, + true + ); + const result = await processor.process(input); + expect(result.archive.bytes).toBeGreaterThan(4.5 * 1024 * 1024); + expect(result.icons).toEqual([]); + expect(JSON.stringify(result).length).toBeLessThan(1024); + expect(await sha256(f.objects.get(f.key(input.archive))!.bytes)).toBe( + result.archive.sha256 + ); + const corrected = await processor.process({ + ...input, + manifest: { ...manifest, description: "Rebuilt after a large package" }, + archive: crypto.randomUUID(), + icons: undefined + }); + expect(corrected.archive.sha256).not.toBe(result.archive.sha256); + } finally { + server.closeAllConnections(); + await new Promise((resolve, reject) => + server.close((error) => (error ? reject(error) : resolve())) + ); + } + }, 30000); + it("fails closed on malformed runner responses and leaves timeouts retryable", async () => { + const input = await fixture().input(await archive()); + const processor = new HttpPackageProcessor( + "https://processor.example.test/process", + "secret" + ); + for (const result of [ + { archive: { bytes: -1, sha256: "0".repeat(64) }, icons: [] }, + { + archive: { bytes: 20, sha256: "0".repeat(64) }, + icons: [ + { + id: crypto.randomUUID(), + format: "svg", + bytes: 10, + sha256: "0".repeat(64) + } + ] + } + ]) { + vi.stubGlobal("fetch", async () => Response.json({ result })); + await expect(processor.process(input)).rejects.toMatchObject({ + status: 503 + }); + } + const controller = new AbortController(); + vi.spyOn(AbortSignal, "timeout").mockReturnValue(controller.signal); + vi.stubGlobal( + "fetch", + (request: Request) => + new Promise((_resolve, reject) => + request.signal.addEventListener( + "abort", + () => reject(request.signal.reason), + { once: true } + ) + ) + ); + const timedOut = expect(processor.process(input)).rejects.toMatchObject({ + status: 503 + }); + controller.abort(new DOMException("Processor timed out", "TimeoutError")); + await timedOut; + }); +}); diff --git a/services/extension-packager/tsconfig.json b/services/extension-packager/tsconfig.json new file mode 100644 index 00000000..e0d52a35 --- /dev/null +++ b/services/extension-packager/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "../../tsconfig.json", + "include": [ + "src/**/*.ts", + "test/**/*.ts", + "netlify/**/*.ts", + "scripts/**/*.ts" + ] +} diff --git a/src/lib/adapters/cloudflare/package-images.ts b/src/lib/adapters/cloudflare/package-images.ts deleted file mode 100644 index 9b08c0ea..00000000 --- a/src/lib/adapters/cloudflare/package-images.ts +++ /dev/null @@ -1,62 +0,0 @@ -import { sanitizeSvg } from "@zeropress/svg-hush/workerd"; -import type { - PackageImageProcessor, - IconVariant -} from "../../../services/extensions/v2/packages/images"; -import { PackageError } from "../../../services/extensions/v2/packages/errors"; -import { boundedBytes } from "../../../services/extensions/v2/packages/store"; - -export class CloudflarePackageImages implements PackageImageProcessor { - constructor(private readonly images?: ImagesBinding) {} - async prepare( - icon: Uint8Array, - isSvg: boolean - ): Promise { - const images = this.images; - if (isSvg) { - try { - return [{ format: "svg", bytes: new Uint8Array(sanitizeSvg(icon)) }]; - } catch { - throw new PackageError( - "INVALID_ICON", - "Package SVG icon could not be sanitized", - 422 - ); - } - } - if (!images) - throw new PackageError( - "PACKAGES_DISABLED", - "Image processing is not configured", - 503 - ); - const variants: IconVariant[] = []; - try { - const info = await images.info(new Blob([icon]).stream()); - // Check decoded format too: a misleading filename must not bypass sanitization. - if (info.format === "image/svg+xml") - return [{ format: "svg", bytes: new Uint8Array(sanitizeSvg(icon)) }]; - for (const format of ["avif", "png"] as const) { - const output = await images - .input(new Blob([icon]).stream()) - .transform({ - width: 256, - height: 256, - fit: "scale-down" - }) - .output({ format: `image/${format}`, anim: false }); - variants.push({ - format, - bytes: await boundedBytes(output.image(), 1024 * 1024) - }); - } - } catch { - throw new PackageError( - "INVALID_ICON", - "Package icon could not be processed", - 422 - ); - } - return variants; - } -} diff --git a/test/tsconfig.json b/test/tsconfig.json deleted file mode 100644 index 5e1bd461..00000000 --- a/test/tsconfig.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "extends": "../tsconfig.json", - "compilerOptions": { - "moduleResolution": "bundler", - "types": ["@cloudflare/vitest-plugin"] - }, - "include": ["./**/*.ts", "../worker-configuration.d.ts"] -} diff --git a/tsconfig.json b/tsconfig.json index 7583443e..2eb32ffb 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -6,10 +6,7 @@ "strict": true, "skipLibCheck": true, "lib": ["ESNext"], - "types": [ - "./worker-configuration.d.ts", - "@cloudflare/vitest-plugin", - "node" - ] - } + "types": ["node"] + }, + "include": ["*.ts"] }