diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 283016ba..6082cef6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -100,6 +100,7 @@ jobs: deployment: false concurrency: group: api-migrations-${{ github.ref == 'refs/heads/main' && 'production' || 'preview' }} + queue: max cancel-in-progress: false steps: - name: Checkout Code @@ -163,6 +164,53 @@ jobs: CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} CLOUDFLARE_ACCOUNT_ID: ${{ vars.CLOUDFLARE_ACCOUNT_ID }} + staging: + name: Deploy Staging API + needs: [validation, build] + if: github.event_name == 'push' && github.ref == 'refs/heads/main' && vars.API_PREVIEW_RESOURCES != '' + runs-on: ubuntu-latest + timeout-minutes: 15 + environment: + name: preview + deployment: false + concurrency: + group: api-migrations-preview + queue: max + cancel-in-progress: false + steps: + - name: Checkout Code + uses: actions/checkout@0c366fd6a839edf440554fa01a7085ccba70ac98 + with: + persist-credentials: false + - name: Setup Node.js + uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f + with: + node-version: "26.13.1" + cache: npm + - name: Install Dependencies + run: npm ci + - name: Download Deployment Artifact + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 + with: + name: api-worker-build + path: services/api/dist/ + - name: Apply Staging Migrations and Deploy + id: staging + env: + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + CLOUDFLARE_ACCOUNT_ID: ${{ vars.CLOUDFLARE_ACCOUNT_ID }} + run: | + for binding in DB_CENTRAL_ALERTS DB_EXTENSIONS; do + npm exec -- wrangler d1 migrations apply "$binding" --remote --config services/api/dist/wrangler.preview-migrations.json + done + WRANGLER_OUTPUT_FILE_PATH="$RUNNER_TEMP/staging-deployment.jsonl" npm exec -- wrangler preview --config services/api/dist/wrangler.json --name staging --message "main $GITHUB_SHA" + url=$(jq -er 'select(.type == "preview") | .preview_urls[0]' "$RUNNER_TEMP/staging-deployment.jsonl") + printf 'url=%s\n' "$url" >> "$GITHUB_OUTPUT" + - name: Check Staging API + env: + STAGING_API_URL: ${{ steps.staging.outputs.url }} + run: curl --fail --silent --show-error --max-time 30 --retry 3 --retry-delay 2 --output /dev/null "$STAGING_API_URL/extensions/v2/extensions" + deploy: name: Deploy Production needs: [validation, build, migrations] diff --git a/services/api/AGENTS.md b/services/api/AGENTS.md index 4d44bd0f..b7fa2d3c 100644 --- a/services/api/AGENTS.md +++ b/services/api/AGENTS.md @@ -69,7 +69,7 @@ Each service documents its own contract and operational detail in its own `READM - Bindings, cron, and vars are defined in `wrangler.jsonc` (D1, KV, R2, Durable Objects, rate limiters, service bindings); keep names aligned with `CloudflareBindings`. - Use Wrangler secrets for production tokens instead of committing them. - `ASSERTION_SIGNING_SECRET`: HMAC key the extensions v2 API uses to verify short-lived bearer assertions minted by the extensions site (`src/lib/auth/bearer-assertion.ts`). Not sent over the wire — only signs/verifies server-side in each Worker. Add `ASSERTION_SIGNING_SECRET="..."` to `.dev.vars` for local dev; set via `wrangler secret put ASSERTION_SIGNING_SECRET` in production, matching the value configured in the extensions site's Worker. -- `EXTENSIONS_REVALIDATE_SECRET`: bearer token the extensions v2 API sends to the extensions site's `POST /api/revalidate` (over the `EXTENSIONS_FRONTEND` service binding) to purge the site's CDN-cached extension pages after content mutations. Must match the value configured in the extensions site's Worker. Same local/prod setup as `ASSERTION_SIGNING_SECRET`. +- `EXTENSIONS_REVALIDATE_SECRET`: bearer token the extensions v2 API sends to the directory's `POST /api/revalidate` to purge cached pages after content mutations. Production uses the `EXTENSIONS_FRONTEND` service binding; Previews use HTTP to `EXTENSIONS_FRONTEND_URL`, configured in `wrangler.jsonc`. The secret must match the target directory deployment. ## Cache Revalidation diff --git a/services/api/README.md b/services/api/README.md index 734879f4..e43df8bd 100644 --- a/services/api/README.md +++ b/services/api/README.md @@ -81,7 +81,7 @@ We use [Cloudflare D1](https://developers.cloudflare.com/d1/) and [KV](https://d - **R2 Bucket** (`DOWNLOAD_BUCKET`): Shared by versions, previews, and stats for release/preview zips — see [`src/services/previews/v1/README.md`](src/services/previews/v1/README.md) and the comment in `wrangler.jsonc` for which bucket this points at and why. - **Durable Object** (`PREVIEW_GITHUB_BUDGET`): Token bucket limiting previews GitHub calls. Policy lives in `src/services/previews/v1/budget.ts`. - **Rate limiters** (`EXTENSION_WRITE_RATE_LIMITER`, `PROFILE_CREATION_RATE_LIMITER`): Edge pacing for extension writes and profile creation. Durable quotas are enforced in D1; see [extensions v2 README](src/services/extensions/v2/README.md#limits-and-budgets). -- **Service binding** (`EXTENSIONS_FRONTEND`): Private binding to the Extensions site worker, used to purge cached extension pages after mutations. +- **Service binding** (`EXTENSIONS_FRONTEND`): Production-only private binding to the Extensions site Worker, used to purge cached extension pages after mutations. Previews use HTTP instead (see `EXTENSIONS_FRONTEND_URL` below). - **Cron** (`0 * * * *`): Hourly extension resource maintenance. See [extensions v2 README](src/services/extensions/v2/README.md#revisions-and-retention). ### Environment Variables @@ -102,7 +102,8 @@ Package storage configuration is listed in [PACKAGING.md](src/services/extension - `GITHUB_APP_PRIVATE_KEY`: GitHub's downloaded App private key (PEM), stored as a secret. - `ASSERTION_SIGNING_SECRET`: Shared HMAC secret used to verify the short-lived bearer assertions minted by the Extensions site. Configure the same value in both Workers; it is never sent to clients. - `ASSERTION_SIGNING_SECRET_PREVIOUS`: Optional previous HMAC secret accepted during a signing-key rotation. -- `EXTENSIONS_REVALIDATE_SECRET`: Bearer token the API sends to the Extensions site's `POST /api/revalidate` (over the `EXTENSIONS_FRONTEND` service binding) to purge the site's CDN-cached extension pages after content mutations. Configure the same value in both Workers; it is never sent to clients. +- `EXTENSIONS_REVALIDATE_SECRET`: Bearer token used for the Extensions site's `POST /api/revalidate`, sent through the `EXTENSIONS_FRONTEND` service binding in production or over HTTP in Previews. Configure the same value in the API and its target directory deployment; it is never sent to browser clients. +- `EXTENSIONS_FRONTEND_URL`: Preview directory origin used for HTTP cache invalidation when there is no `EXTENSIONS_FRONTEND` service binding. Set in `wrangler.jsonc` to the named staging directory Preview; generated Preview deployments omit the production binding. - `EXTENSIONS_EMAIL_PROVIDER`: Production secret managed in the Worker dashboard. Local development sets `disabled` in `.dev.vars`; Previews set `disabled` in `wrangler.jsonc`. Use `mxroute` or `resend` to send moderator notification emails, or unset/`disabled` to skip sending. - `EXTENSIONS_EMAIL_FROM`: Sender address for notification emails (default `noreply@fossbilling.org`). - `EXTENSIONS_EMAIL_REPLY_TO`: Optional Reply-To address for a monitored inbox. Omit when unset (no Reply-To header is sent) — notification emails do not invite replies. @@ -219,8 +220,8 @@ Migration jobs serialize access to the shared staging databases. environment secrets with trusted deployment approval instead. That is a different access policy, not a requirement of this trusted-maintainer setup. -2. Provision staging D1 databases, KV namespaces, a download bucket and a separately - deployed staging Extensions frontend. Set the repository variable `API_PREVIEW_RESOURCES` +2. Provision staging D1 databases, KV namespaces and a download bucket. Set the repository + variable `API_PREVIEW_RESOURCES` to their identifiers, using the following JSON shape: ```json @@ -229,8 +230,7 @@ Migration jobs serialize access to the shared staging databases. "DB_CENTRAL_ALERTS": "", "AUTH_KV": "", "CACHE_KV": "", - "DOWNLOAD_BUCKET": "", - "EXTENSIONS_FRONTEND": "" + "DOWNLOAD_BUCKET": "" } ``` @@ -242,8 +242,8 @@ Migration jobs serialize access to the shared staging databases. 3. Configure Preview runtime secrets in the API Worker's **Previews Base** settings. Use separate assertion-signing and revalidation secrets, matching both with - the staging frontend. Set all three `GITHUB_APP_*` values, selecting the preview - private key (see [GitHub App authentication](#github-app-authentication)). Set M2M issuer, + the directory's named `staging` Preview. Set all three `GITHUB_APP_*` values, + selecting the preview private key (see [GitHub App authentication](#github-app-authentication)). Set M2M issuer, JWKS and audience variables there when using a separate test issuer. Operational production variables and secrets remain dashboard-managed with `keep_vars: true`. 4. Configure required checks for the validation/build jobs, then disable the @@ -251,12 +251,18 @@ Migration jobs serialize access to the shared staging databases. The workflow compiles the branch's Preview settings into the artifact. Each Preview receives isolated Durable Object state; the data bindings above are shared staging -resources. A frontend service binding calls that Worker's normal deployment, so use a -staging Worker name rather than the production Extensions worker. Configure that -frontend with HTTP API transport and a Preview API URL, without production routes -or API bindings. A long-lived Preview such as `staging` can back this frontend; -refresh it from the prepared artifact with `npm run cf-preview --workspace @fossbilling/api -- --name staging`. -Register its OAuth callback with a dedicated test client before testing sign-in. +resources. The directory's named `staging` Preview at +`https://staging-extensions.fossbilling.workers.dev` uses HTTP API transport. +Preview API cache invalidations also use HTTP (`EXTENSIONS_FRONTEND_URL`), +because service bindings cannot target named Previews; production retains its +private frontend service binding. The fixed API `staging` Preview at +`https://staging-api.fossbilling.workers.dev` is refreshed on pushes to `main`, after +validation and the artifact build. Its job applies staging migrations, deploys the +same artifact with Preview bindings, and checks the public extensions endpoint +using the Preview URL returned by Wrangler. +Here `staging` is a named Preview, not a Git branch; its source is `main`. +Register the directory's staging OAuth callback with the dedicated preview client +before testing sign-in. Cloudflare Previews cannot consume Queues or receive Cron Triggers. Their configuration therefore disables packaging and notification email, and omits Queue bindings. Full diff --git a/services/api/scripts/prepare-deployment.ts b/services/api/scripts/prepare-deployment.ts index da9851ff..f46d5e65 100644 --- a/services/api/scripts/prepare-deployment.ts +++ b/services/api/scripts/prepare-deployment.ts @@ -10,13 +10,12 @@ import { fileURLToPath } from "node:url"; import { experimental_readRawConfig, type Unstable_RawConfig } from "wrangler"; import { z } from "zod"; -const PreviewResourcesSchema = z.strictObject({ +const PreviewResourcesSchema = z.object({ DB_EXTENSIONS: z.uuid().toLowerCase(), DB_CENTRAL_ALERTS: z.uuid().toLowerCase(), AUTH_KV: z.string().regex(/^[a-f0-9]{32}$/), CACHE_KV: z.string().regex(/^[a-f0-9]{32}$/), - DOWNLOAD_BUCKET: z.string().min(1), - EXTENSIONS_FRONTEND: z.string().min(1) + DOWNLOAD_BUCKET: z.string().min(1) }); // Keep the bundle, runtime bindings and migration targets in one deployable artifact. @@ -32,8 +31,7 @@ export function prepareDeployment(root: string, previewResources?: string) { const production = new Set([ ...(config.d1_databases ?? []).map((value) => value.database_id), ...(config.kv_namespaces ?? []).map((value) => value.id), - ...(config.r2_buckets ?? []).map((value) => value.bucket_name), - ...(config.services ?? []).map((value) => value.service) + ...(config.r2_buckets ?? []).map((value) => value.bucket_name) ]); const values = Object.values(resources); if ( @@ -85,12 +83,7 @@ export function prepareDeployment(root: string, previewResources?: string) { r2_buckets: [ { binding: "DOWNLOAD_BUCKET", bucket_name: resources.DOWNLOAD_BUCKET } ], - services: [ - { - binding: "EXTENSIONS_FRONTEND", - service: resources.EXTENSIONS_FRONTEND - } - ] + services: [] }; writeFileSync( join(output, "wrangler.preview-migrations.json"), diff --git a/services/api/src/lib/adapters/cloudflare/index.ts b/services/api/src/lib/adapters/cloudflare/index.ts index cdac7578..5c305ffb 100644 --- a/services/api/src/lib/adapters/cloudflare/index.ts +++ b/services/api/src/lib/adapters/cloudflare/index.ts @@ -18,6 +18,10 @@ export function createCloudflareBindings( ): IPlatformBindings { let bindings = bindingsCache.get(env); if (!bindings) { + const environment = new CloudflareEnvironmentAdapter( + env as unknown as Record + ); + const frontendUrl = environment.get("EXTENSIONS_FRONTEND_URL"); bindings = { queues: env.EXTENSION_PACKAGE_QUEUE ? { @@ -26,9 +30,25 @@ export function createCloudflareBindings( ) } : {}, - services: { - EXTENSIONS_FRONTEND: (request) => env.EXTENSIONS_FRONTEND.fetch(request) - }, + services: env.EXTENSIONS_FRONTEND + ? { + EXTENSIONS_FRONTEND: (request) => + env.EXTENSIONS_FRONTEND.fetch(request) + } + : frontendUrl + ? { + // Named Previews cannot be targeted by a service binding. + EXTENSIONS_FRONTEND: (request) => { + const url = new URL(request.url); + return globalThis.fetch( + new Request( + new URL(url.pathname + url.search, frontendUrl), + request + ) + ); + } + } + : {}, rateLimiters: { EXTENSION_WRITE_RATE_LIMITER: async (key) => (await env.EXTENSION_WRITE_RATE_LIMITER.limit({ key })).success, @@ -39,9 +59,7 @@ export function createCloudflareBindings( CACHE_KV: new CloudflareKVAdapter(env.CACHE_KV), AUTH_KV: new CloudflareKVAdapter(env.AUTH_KV) }, - environment: new CloudflareEnvironmentAdapter( - env as unknown as Record - ) + environment }; bindingsCache.set(env, bindings); } diff --git a/services/api/test/deployment.node.test.ts b/services/api/test/deployment.node.test.ts index 68bf68f3..1383d4a3 100644 --- a/services/api/test/deployment.node.test.ts +++ b/services/api/test/deployment.node.test.ts @@ -20,8 +20,7 @@ const resources = { DB_CENTRAL_ALERTS: "22222222-2222-4222-8222-222222222222", AUTH_KV: "a".repeat(32), CACHE_KV: "b".repeat(32), - DOWNLOAD_BUCKET: "api-staging", - EXTENSIONS_FRONTEND: "api-staging" + DOWNLOAD_BUCKET: "api-staging" }; const folders: string[] = []; afterEach(() => { @@ -74,9 +73,11 @@ describe("API deployment artifact", () => { expect(artifact.previews.r2_buckets).toEqual([ { binding: "DOWNLOAD_BUCKET", bucket_name: resources.DOWNLOAD_BUCKET } ]); - expect(artifact.previews.services).toEqual([ - { binding: "EXTENSIONS_FRONTEND", service: resources.EXTENSIONS_FRONTEND } - ]); + expect(artifact.previews.services).toEqual([]); + expect(artifact.services).toEqual(source.services); + expect(artifact.previews.vars.EXTENSIONS_FRONTEND_URL).toBe( + "https://staging-extensions.fossbilling.workers.dev" + ); expect(artifact.previews.vars.EXTENSION_PACKAGES_MODE).toBe("disabled"); for (const database of source.d1_databases!) { const packaged = artifact.d1_databases.find( @@ -112,7 +113,6 @@ describe("API deployment artifact", () => { { AUTH_KV: source.kv_namespaces![0].id }, { CACHE_KV: source.kv_namespaces![1].id }, { DOWNLOAD_BUCKET: source.r2_buckets![0].bucket_name }, - { EXTENSIONS_FRONTEND: source.services![0].service }, { AUTH_KV: resources.CACHE_KV } ]; for (const override of overrides) { diff --git a/services/api/test/lib/adapters/cloudflare/index.test.ts b/services/api/test/lib/adapters/cloudflare/index.test.ts new file mode 100644 index 00000000..efd4ca0b --- /dev/null +++ b/services/api/test/lib/adapters/cloudflare/index.test.ts @@ -0,0 +1,44 @@ +import { afterEach, expect, it, vi } from "vitest"; +import { createCloudflareBindings } from "../../../../src/lib/adapters/cloudflare/index"; + +afterEach(() => vi.unstubAllGlobals()); + +it("uses the service binding for production and HTTP for the named staging Preview", async () => { + const bindingFetch = vi.fn( + async (_request: Request) => new Response(null, { status: 204 }) + ); + const httpFetch = vi.fn( + async (_request: Request) => new Response(null, { status: 204 }) + ); + vi.stubGlobal("fetch", httpFetch); + const request = () => + new Request("https://extensions.fossbilling.org/api/revalidate", { + method: "POST", + headers: { + authorization: "Bearer staging-secret", + "content-type": "application/json" + }, + body: JSON.stringify({ tags: ["extensions"] }) + }); + const production = createCloudflareBindings({ + EXTENSIONS_FRONTEND: { fetch: bindingFetch } + } as unknown as CloudflareBindings); + await production.services!.EXTENSIONS_FRONTEND(request()); + expect(bindingFetch).toHaveBeenCalledOnce(); + expect(httpFetch).not.toHaveBeenCalled(); + + const staging = createCloudflareBindings({ + EXTENSIONS_FRONTEND_URL: + "https://staging-extensions.fossbilling.workers.dev" + } as unknown as CloudflareBindings); + await staging.services!.EXTENSIONS_FRONTEND(request()); + expect(httpFetch).toHaveBeenCalledOnce(); + const forwarded = httpFetch.mock.calls[0][0] as Request; + expect(forwarded.url).toBe( + "https://staging-extensions.fossbilling.workers.dev/api/revalidate" + ); + expect(forwarded.method).toBe("POST"); + expect(forwarded.headers.get("authorization")).toBe("Bearer staging-secret"); + expect(await forwarded.json()).toEqual({ tags: ["extensions"] }); + expect(bindingFetch).toHaveBeenCalledOnce(); +}); diff --git a/services/api/wrangler.jsonc b/services/api/wrangler.jsonc index 66303011..65f6dba2 100644 --- a/services/api/wrangler.jsonc +++ b/services/api/wrangler.jsonc @@ -121,6 +121,7 @@ "previews": { "vars": { "ENVIRONMENT": "preview", + "EXTENSIONS_FRONTEND_URL": "https://staging-extensions.fossbilling.workers.dev", "EXTENSION_PACKAGES_MODE": "disabled", "EXTENSIONS_EMAIL_PROVIDER": "disabled", "EXTENSIONS_RETENTION_MODE": "dry-run"